def instancenorm_3_constraints(pattern): epsilon_var = _get_var(pattern.add_epsilon, pattern.mean1) gamma_var = mb.const( val=np.ones(shape=(1, pattern.root_var.shape[1], 1, 1)), name="gamma_var" ) beta_var = mb.const( val=np.zeros(shape=(1, pattern.root_var.shape[1], 1, 1)), name="_fuse_layernorm_or_instancenorm_beta", ) passed = True passed = passed and _check_reduce_op(pattern.main_reduce) passed = passed and pattern.sub.x == pattern.root_var and pattern.sub.y == pattern.main_reduce.outputs[0] passed = passed and _check_reduce_op(pattern.mean1) passed = passed and pattern.mul_sub.y.val is not None and pattern.mul_sub.y.val == -1 passed = passed and _general_constraints(pattern, epsilon_var, gamma_var, beta_var) passed = passed and _instancenorm_constraints(pattern) return passed
def linear_prog(input): W = mb.const(val=np.random.rand(100, 5000), name="const_W") out = mb.linear(x=input, weight=W, name="output") return out
def transform_pattern(pattern): bias_value = _get_bias_var(pattern).val is_conv_op = (pattern.conv.op_type == "conv") is_bias_scalar = False if not isinstance(bias_value, np.ndarray): is_bias_scalar = True bias_value = np.array([bias_value ]) if is_bias_scalar else np.squeeze(bias_value) if pattern.add_or_sub.op_type == "sub": bias_value *= -1 # everything looks good, now find the new updated bias old_bias = pattern.conv.inputs.get("bias", None) old_bias_value = None if old_bias is not None and old_bias.val is not None: old_bias_value = old_bias.val if old_bias is None: # need to create a fresh numpy array for bias if np.prod(bias_value.shape) == 1: # its a scalar bias # need to find the value of Cout to form a new bias # conv_transpose has weight format [K, C_out, spatial dims] # conv has weight format [C_out, K, spatial dims] Cout = pattern.conv.weight.val.shape[0 if is_conv_op else 1] new_bias_value = np.broadcast_to(bias_value, (Cout, )) else: new_bias_value = bias_value else: # just need to update the existing bias array new_bias_value = old_bias_value + bias_value # create a new conv op with the new bias value, copying rest of the attributes out_name = pattern.add_or_sub.outputs[0].name if new_bias_value.dtype != np.float32 and new_bias_value.dtype != np.float16: # cast the bias to match the weight type weight_np_type = types.nptype_from_builtin( pattern.conv.inputs["weight"].sym_type.get_primitive()) logging.warning( "conv_bias_fusion pass: casting bias " "from {} to {} to match the dtype of the weight of the conv layer". format(new_bias_value.dtype, weight_np_type)) new_bias_value = new_bias_value.astype(weight_np_type) new_bias_var = mb.const(val=new_bias_value, before_op=pattern.conv) conv_kargs = { "bias": new_bias_var, "name": out_name, "before_op": pattern.conv } for k, v in pattern.conv.inputs.items(): if k == "bias": continue conv_kargs[k] = v if is_conv_op: x = mb.conv(**conv_kargs) else: x = mb.conv_transpose(**conv_kargs) pattern.add_or_sub.enclosing_block.replace_uses_of_var_after_op( anchor_op=pattern.add_or_sub, old_var=pattern.add_or_sub.outputs[0], new_var=x) # Remove all the ops at once pattern.block.remove_ops(pattern.op_list())