def infer_symbolic_relations(self, n: Node):
if HAS_SYMPY:
n.type = self.convert_to_sympy_symbols(n.type)
if n.op == 'call_function':
if n.target in _RULES:
return _RULES[n.target](n)
else:
pass
if n.op == 'call_module':
module_instance = self.traced.get_submodule(n.target)
if type(module_instance) in _RULES:
return _RULES[type(module_instance)](n, module_instance)
else:
pass
if n.op == 'output':
def get_node_type(a):
return a.type
n.type = torch.fx.node.map_arg(n.args[0], get_node_type)
return n.type
else:
pass
else:
pass
def add_inference_rule(n: Node):
assert isinstance(n.args[0], Node)
assert isinstance(n.args[1], Node)
t1 = n.args[0].type
t2 = n.args[1].type
# handle scalar addition
if t1 == int and isinstance(t2, TensorType):
n.type = t2
return n.type
elif t2 == int and isinstance(t1, TensorType):
n.type = t1
return n.type
(new_t1, new_t2) = broadcast_types(t1, t2)
n.args[0].type = new_t1
n.args[1].type = new_t2
if is_consistent(new_t1, new_t2):
# we return the more precise type
if is_more_precise(new_t1, new_t2):
n.type = new_t2
else:
n.type = new_t1
return n.type
else:
raise TypeError(f'Cannot add arguments {n.args[0]} ({ n.args[0].type}) and {n.args[1]} ({ n.args[1].type}) in node {n}.'
f' Types should match ')
def refine_node(self, n: Node):
"""
Returns a list of equality constraints for
call_module and call_function nodes.
Models the relation between input and output dimensions
using constraints in case they are both tensors.
All operations used in resnet50 are defined.
"""
if n.type is None:
n.type = Dyn
n.type = self.replace_dyn_with_fresh_var(n.type)
if n.op == 'call_function':
if n.target in _REFINEMENT_RULES:
self.constraints += _REFINEMENT_RULES[n.target](n)
else:
pass
if n.op == 'call_module':
module_instance = self.traced.get_submodule(n.target)
if type(module_instance) in _REFINEMENT_RULES:
self.constraints += _REFINEMENT_RULES[type(module_instance)](n)
else:
pass
if n.op == 'output':
assert isinstance(n.args[0], Node)
n.type = n.args[0].type
else:
pass
def run_node(self, n : Node) -> Any:
try:
result = super().run_node(n)
except Exception:
traceback.print_exc()
raise RuntimeError(
f"ShapeProp error for: node={n.format_node()} with "
f"meta={n.meta}"
)
found_tensor = False
def extract_tensor_meta(obj):
if isinstance(obj, torch.Tensor):
nonlocal found_tensor
found_tensor = True
return _extract_tensor_metadata(obj)
else:
return obj
meta = map_aggregate(result, extract_tensor_meta)
if found_tensor:
n.meta['tensor_meta'] = meta
n.meta['type'] = type(result)
return result
def transpose_inference_rule(n: Node):
"""
We check that dimentions for the transpose operations
are within range of the tensor type of the node
"""
if n.target == torch.transpose:
assert isinstance(n.args[0], Node)
t = n.args[0].type
assert isinstance(n.args[1], int)
assert isinstance(n.args[2], int)
dim1, dim2 = n.args[1], n.args[2]
if t == Dyn:
n.type = Dyn
return n.type
elif isinstance(t, TensorType):
if 0 <= dim1 < len(t.__args__) and 0 <= dim2 < len(t.__args__):
new_type = list(t.__args__)
new_type[dim1], new_type[dim2] = new_type[dim2], new_type[dim1]
final = TensorType(new_type)
n.type = get_greatest_upper_bound(n.type, final)
return n.type
else:
raise TypeError(
f'Cannot transpose {dim1} and {dim2} in type {t} for node {n}'
)
else:
raise TypeError(
f'Cannot transpose {dim1} and {dim2} in type {t} for node {n}')
def transpose_inference_rule(n: Node):
if n.target == torch.transpose:
assert isinstance(n.args[0], Node)
t = n.args[0].type
assert isinstance(n.args[1], int)
assert isinstance(n.args[2], int)
dim1, dim2 = n.args[1], n.args[2]
if t == Dyn:
n.type = Dyn
return n.type
elif isinstance(t, TensorType):
if 0 <= dim1 < len(t.__args__) and 0 <= dim2 < len(t.__args__):
new_type = list(t.__args__)
new_type[dim1], new_type[dim2] = new_type[dim2], new_type[dim1]
final = TensorType(new_type)
n.type = final
return n.type
else:
raise TypeError(f'Cannot transpose {dim1} and {dim2} in type {t} for node {n}')
else:
raise TypeError(f'Cannot transpose {dim1} and {dim2} in type {t} for node {n}')
def type_check_node(self, n: Node):
"""
Type check a given fx node.
Current operations:
- Reshape
- Transpose
- Add
"""
if n.op == 'placeholder':
if n.type is None:
n.type = Dyn
else:
return n.type
if n.op == 'call_function':
if n.target in _INFERENCE_RULES:
return _INFERENCE_RULES[n.target](n)
else:
raise RuntimeError(f'No inference rule registered for target {n.target}!')
if n.op == 'output':
assert isinstance(n.args[0], Node)
n.type = n.args[0].type
return n.type
else:
raise NotImplementedError("Method not yet implemented")
def bn2d_inference_rule(n: Node, module_instance):
"""
Given a BatchNorm2D instance and a node check the following conditions:
- the input type can be expanded to a size 4 tensor: t = (x_1, x_2, x_3, x_4)
- the current node type can be expanded to a size 4 tensor: t' = (x_1', x_2', x_3', x_4')
- t is consistent with t'
- x_2 is consistent with the module's num_features
- x_2' is consistent with the module's num_features
output type: the more precise type of t and t'
"""
assert isinstance(n.args[0], Node)
n.args[0].type = expand_to_tensor_dim(n.args[0].type, 4)
arg_type = n.args[0].type
n.type = expand_to_tensor_dim(n.type, 4)
# we check the conditions on the incoming argument
# and any existing annotation
# we also check for consistency between both annotations
if is_consistent(arg_type.__args__[1], module_instance.num_features) and \
is_consistent(n.type.__args__[1], module_instance.num_features) and \
is_consistent(arg_type, n.type):
# we choose the more precise type
# to be the node type
# so if an incoming argument has more type information
# we set this node's type to be the argument type
n.type = get_greatest_upper_bound(arg_type, n.type)
return n.type
else:
raise TypeError(f'Cannot apply {module_instance} with input type {arg_type} and existing type {n.type} on {n}')
def run_node(self, n: Node) -> Any:
result = super().run_node(n)
if isinstance(result, torch.Tensor):
n.shape = result.shape # type: ignore
n.dtype = result.dtype # type: ignore
return result
def type_check_node(self, n: Node):
"""
Type check a given fx node.
Current operations:
- Reshape
- Transpose
- Add
- Relu
- conv2d
- batchnorm2d
- flatten
- maxpool2d
- adaptiveavgpool2d
- linear
"""
if n.type is None:
n.type = Dyn
if n.op == 'placeholder':
return n.type
elif n.op == 'get_attr':
t = get_parameter(self.traced, n.target) # type: ignore[arg-type]
if isinstance(t.data, torch.Tensor):
n.type = TensorType(t.data.shape)
return n.type
elif n.op == 'call_function':
if n.target == getattr:
assert getattr in _INFERENCE_RULES
return _INFERENCE_RULES[n.target](n, self.traced)
elif n.target in _INFERENCE_RULES:
return _INFERENCE_RULES[n.target](n)
else:
raise RuntimeError(
f'No inference rule registered for target {n.target}!')
elif n.op == 'call_module':
module_instance = self.traced.get_submodule(n.target)
if type(module_instance) in _INFERENCE_RULES:
return _INFERENCE_RULES[type(module_instance)](n,
module_instance)
else:
raise RuntimeError(
f'No inference rule registered for class {type(module_instance)}!'
)
elif n.op == 'output':
def get_node_type(a):
return a.type
n.type = torch.fx.node.map_arg(n.args[0], get_node_type)
return n.type
else:
raise NotImplementedError(f"Method {n.op} not yet implemented")
def run_node(self, n: Node) -> Any:
result = super().run_node(n)
if isinstance(result, torch.Tensor):
n.meta['shape'] = result.shape
n.meta['dtype'] = result.dtype
n.meta['stride'] = result.stride()
n.meta['is_quantized'] = result.is_quantized
memory_formats = {
torch.contiguous_format,
torch.channels_last,
torch.channels_last_3d,
}
memory_format = None
for query_format in memory_formats:
if result.is_contiguous(memory_format=query_format):
memory_format = query_format
break
n.meta['memory_format'] = memory_format
if n.meta['is_quantized']:
n.meta['qscheme'] = result.qscheme()
if n.meta['qscheme'] in {
torch.per_tensor_affine, torch.per_tensor_symmetric
}:
n.meta['q_scale'] = result.q_scale()
n.meta['q_zero_point'] = result.q_zero_point()
return result
def flatten_inference_rule(n: Node):
"""
Applies the flatten shape information to the input then gets the
greatest upper bound of the resulting type and the existing type
"""
assert isinstance(n.args[0], Node)
# set the default start and end dims
start_dim = 1
end_dim = -1
if len(n.args) > 1:
assert isinstance(n.args[1], int)
start_dim = n.args[1]
if len(n.args) > 2:
assert isinstance(n.args[2], int)
end_dim = n.args[2]
if n.args[0].type == Dyn and isinstance(n.type, TensorType):
n.args[0].type = expand_to_tensor_dim(n.args[0].type,
len(n.type.__args__))
if isinstance(n.args[0].type, TensorType):
output_type = flatten_check(n.args[0].type, start_dim, end_dim)
n.type = get_greatest_upper_bound(output_type, n.type)
return n.type
def conv2d_inference_rule(n: Node, module_instance):
"""
Given a Conv2D instance and a node check the following conditions:
- the input type can be expanded to a size 4 tensor: t = (x_1, x_2, H, W)
- the current node type can be expanded to a size 4 tensor: t' = (x_1', x_2', x_3', x_4')
- x_2 is consistent with the module's in_channels
- let o = (x_1, out_channels, H_out, W_out)
then the output is the greatest upper bound of o and the existing node type t'.
"""
assert isinstance(n.args[0], Node)
n.args[0].type = expand_to_tensor_dim(n.args[0].type, 4)
arg_type = n.args[0].type
curr_node_type = expand_to_tensor_dim(n.type, 4)
if is_consistent(arg_type.__args__[1], module_instance.in_channels):
w_in = arg_type.__args__[3]
h_in = arg_type.__args__[2]
h_out = calculate_out_dimension(h_in, module_instance, 0)
w_out = calculate_out_dimension(w_in, module_instance, 1)
new_type = TensorType((arg_type.__args__[0], module_instance.out_channels, h_out, w_out))
gub = get_greatest_upper_bound(new_type, curr_node_type)
n.type = gub
return n.type
else:
raise TypeError(f'Cannot apply {module_instance} with input type { arg_type} and existing type {n.type} on {n}')
def run_node(self, n: Node) -> Any:
result = super().run_node(n)
if isinstance(result, torch.Tensor):
n.meta['tensor_meta'] = extract_tensor_metadata(result)
return result
def _nodes_are_equal(self, pn: Node, gn: Node) -> bool:
# TODO: match args and kwargs
# if exact match for placeholder is not required, then use placeholder as a wildcard
if not self.match_placeholder and pn.op == "placeholder":
return True
if pn.target == torch.ops.pseudo.any:
return True
if pn.target == torch.ops.pseudo.oneof:
permissible_targets: List[str] = pn.kwargs.get(
"targets", list()) # type: ignore[assignment]
assert isinstance(permissible_targets, list), \
"pseudo.oneof(permissible_targets=[\"foo\", \"bar\"]) only accept targets as a list"
assert len(
permissible_targets
) > 0, "please specific as least one target for pseudo.oneof"
if gn._pretty_print_target(gn.target) in permissible_targets:
return True
if pn.op == gn.op:
if pn.op == "placeholder" or pn.op == "output":
return True
return pn.target == gn.target
return False
def linear_inference_rule(n: Node, module_instance):
assert isinstance(n.args[0], Node)
if n.args[0].type == Dyn and isinstance(n.type, TensorType):
n.args[0].type = expand_to_tensor_dim(n.args[0].type, len(n.type.__args__))
if isinstance(n.args[0].type, TensorType):
output_type = linear_check(n.args[0].type, module_instance)
n.type = get_greatest_upper_bound(output_type, n.type)
return n.type
def run_node(self, n: Node) -> Any:
result = super().run_node(n)
found_tensor = False
def extract_tensor_meta(obj):
if isinstance(obj, torch.Tensor):
nonlocal found_tensor
found_tensor = True
return extract_tensor_metadata(obj)
else:
return obj
meta = map_aggregate(result, extract_tensor_meta)
if found_tensor:
n.meta['tensor_meta'] = meta
n.meta['type'] = type(result)
return result
def get_attr_inference_rule(n: Node, traced):
attr_node = n.args[0]
attr_name = n.args[1]
if attr_name == "shape":
n.type = Dyn
else:
raise TypeError("Not yet implelemted")
# TODO. We leave it like this till we add a type to represent tensor sizes
return n.type
def reshape_inference_rule(n: Node):
assert isinstance(n.args[0], Node)
t1 = n.args[0].type
assert isinstance(n.args[1], list)
t2 = n.args[1]
t2_type = TensorType([Dyn if elem == -1 else elem for elem in t2])
# if we do not know the original tensor dimension,
# we return the required dimension
if t1 == Dyn:
n.type = t2_type
return t2_type
# if any of the dimensions are unknown,
# we check for divisibility
elif isinstance(t1, TensorType) and Dyn in t1.__args__ or -1 in t2:
assert isinstance(t1, TensorType)
a = [e if e != Dyn else 1 for e in t1.__args__]
p1 = reduce(lambda x, y: x * y, a)
p2 = reduce(lambda x, y: x * y, t2)
if p1 % p2 == 0 or p2 % p1 == 0:
n.type = t2_type
return t2_type
else:
raise TypeError(
f'Cannot reshape in node {n} from {t1} to {t2_type}')
# if all dimensions are known we check the products
elif isinstance(t1, TensorType):
p1 = reduce(lambda x, y: x * y, t1.__args__)
p2 = reduce(lambda x, y: x * y, t2)
if p1 == p2:
n.type = t2_type
return t2_type
else:
raise TypeError(
f'Cannot reshape in node {n} from {t1} to {t2_type}')
else:
raise TypeError(f'Cannot reshape in node {n} from {t1} to {t2_type}')
def type_check_node(self, n: Node):
"""
Type check a given fx node.
Current operations:
- Reshape
- Transpose
- Add
- Relu
- conv2d
- batchnorm2d
- flatten
- maxpool2d
- adaptiveavgpool2d
- linear
"""
if n.type is None:
n.type = Dyn
if n.op == 'placeholder':
return n.type
if n.op == 'call_function':
if n.target in _INFERENCE_RULES:
return _INFERENCE_RULES[n.target](n)
else:
raise RuntimeError(f'No inference rule registered for target {n.target}!')
if n.op == 'call_module':
module_instance = self.traced.get_submodule(n.target)
if type(module_instance) in _INFERENCE_RULES:
return _INFERENCE_RULES[type(module_instance)](n, module_instance)
else:
raise RuntimeError(f'No inference rule registered for class {type(module_instance)}!')
if n.op == 'output':
assert isinstance(n.args[0], Node)
n.type = n.args[0].type
return n.type
else:
raise NotImplementedError("Method not yet implemented")
def adaptiveavgpool2d_inference_rule(n: Node, module_instance):
"""
The input and output sizes should be the same except for the last
two dimensions taken from the input, which represent width and height
"""
assert isinstance(n.args[0], Node)
if n.args[0].type == Dyn and isinstance(n.type, TensorType):
n.args[0].type = expand_to_tensor_dim(n.args[0].type, len(n.type.__args__))
if isinstance(n.args[0].type, TensorType):
output_type = adaptiveavgpool2d_check(n.args[0].type, module_instance)
n.type = get_greatest_upper_bound(n.type, output_type)
return n.type
def conv_rule(n: Node, module_instance):
assert isinstance(n.args[0], Node)
arg_type = n.args[0].type
if isinstance(arg_type, TensorType) and isinstance(n.type, TensorType):
w_in = arg_type.__args__[3]
h_in = arg_type.__args__[2]
h_out = calculate_out_dimension(h_in, module_instance, 0)
w_out = calculate_out_dimension(w_in, module_instance, 1)
new_type = TensorType(
(n.type.__args__[0], n.type.__args__[1], h_out, w_out))
n.type = new_type
return new_type
def relu_inference_rule(n: Node, module_instance):
"""
Input and output shapes should be equal.
"""
assert isinstance(n.args[0], Node)
if n.args[0].type == Dyn and isinstance(n.type, TensorType):
n.args[0].type = expand_to_tensor_dim(n.args[0].type, len(n.type.__args__))
if isinstance(n.args[0].type, TensorType):
n.type = get_greatest_upper_bound(n.args[0].type, n.type)
return n.type
def add_inference_rule(n: Node):
"""
Apply the addition inference rule. This includes:
- scalar addition
- broadcasting semantics
Note that we always return the least precise type between
the operands (after applying broadcasting) to be the final type of the operation
Note that we do not modify the operand types themselves after applying broadcasting
to them. We only use them to calculate the final type
"""
assert isinstance(n.args[0], Node)
assert isinstance(n.args[1], Node)
t1 = n.args[0].type
t2 = n.args[1].type
# handle scalar addition
if t1 == int and isinstance(t2, TensorType):
n.type = t2
return n.type
# handle scalar addition
elif t2 == int and isinstance(t1, TensorType):
n.type = t1
return n.type
# we bring the new types to the point where
# we can check for consistency
# any inconsistency would not have been caused
# by broadcasting at this point
(new_t1, new_t2) = broadcast_types(t1, t2)
if new_t1 != t1 or new_t2 != t2:
n.meta['broadcast'] = True
n.meta[str(n.args[0])] = new_t1
n.meta[str(n.args[1])] = new_t2
else:
n.meta['broadcast'] = False
new_t1 = t1 if not n.meta['broadcast'] else new_t1
new_t2 = t2 if not n.meta['broadcast'] else new_t2
# we check for consistency between the new types
if is_consistent(new_t1, new_t2):
# we return the less precise type because
# broadcasting may have happened
# for operands with shape [1,2,Dyn] and [1,2,1]
# we have to assign the node [1,2,Dyn]
if is_more_precise(new_t1, new_t2):
n.type = new_t2
else:
n.type = new_t1
return n.type
else:
raise TypeError(
f'Cannot add arguments {n.args[0]} ({ n.args[0].type}) and {n.args[1]} ({ n.args[1].type}) in node {n}.'
f' Types should match ')
def linear_inference_rule(n: Node, module_instance):
"""
Applies the shape information to the input then gets the greatest upper bound
of the resulting type and the existing type
"""
assert isinstance(n.args[0], Node)
if n.args[0].type == Dyn and isinstance(n.type, TensorType):
n.args[0].type = expand_to_tensor_dim(n.args[0].type,
len(n.type.__args__))
if isinstance(n.args[0].type, TensorType):
output_type = linear_check(n.args[0].type, module_instance)
n.type = get_greatest_upper_bound(output_type, n.type)
return n.type
def reshape_inference_rule(n: Node):
"""
Without dynamism, the rule checks that the
product of the elements of the argument tensor
type is equal to the product of the elements
of the required shape. We gradualize this rule
by adding a case to handle fully dynamic input
as well as input where some of the tensor dimensions
are unknown. In this case we check for divisibility
"""
assert isinstance(n.args[0], Node)
t1 = n.args[0].type
assert isinstance(n.args[1], list)
t2 = n.args[1]
t2_type = TensorType([Dyn if elem == -1 else elem for elem in t2])
# if we do not know the original tensor dimension,
# we return the required dimension
if t1 == Dyn:
n.type = t2_type
return t2_type
# if any of the dimensions are unknown,
# we check for divisibility
elif isinstance(t1, TensorType):
assert isinstance(t1, TensorType)
a = [e if e != Dyn else 1 for e in t1.__args__]
p1 = reduce(lambda x, y: x * y, a)
p2 = reduce(lambda x, y: x * y, t2)
if p1 % p2 == 0 or p2 % p1 == 0:
n.type = t2_type
return t2_type
else:
raise TypeError(
f'Cannot reshape in node {n} from {t1} to {t2_type}')
else:
raise TypeError(f'Cannot reshape in node {n} from {t1} to {t2_type}')
def conv_rule(n: Node, module_instance):
"""
Represents the outout in terms of an algrbraic expression w.r.t
the input when possible
"""
assert isinstance(n.args[0], Node)
arg_type = n.args[0].type
if isinstance(arg_type, TensorType) and isinstance(n.type, TensorType):
w_in = arg_type.__args__[3]
h_in = arg_type.__args__[2]
h_out = calculate_out_dimension(h_in, module_instance, 0)
w_out = calculate_out_dimension(w_in, module_instance, 1)
new_type = TensorType(
(n.type.__args__[0], n.type.__args__[1], h_out, w_out))
n.type = new_type
return new_type
def get_attr_inference_rule(n: Node, traced):
"""
The current getattr rule only handles the shape attribute
Can be extended to other attributes
The most representitive type we have is "Dyn" but the system
can be extended with more types, such as a type to represent shapes
"""
attr_node = n.args[0]
attr_name = n.args[1]
if attr_name == "shape":
n.type = Dyn
else:
raise TypeError("Not yet implelemted")
# TODO. We leave it like this till we add a type to represent tensor sizes
return n.type
def maxpool2d_inference_rule(n: Node, module_instance):
"""
Given a MaxPool2D instance and a node check the following conditions:
- Input size matches size 3 or 4
- Current node type is consistent with the output type we will calculate
- Input size matches output size and the last two dimensions of the output
are w_out and h_out. The remaining dimensions are the same as the input
- Our final result is the greatest upper bound of the output we calculate
and the current node type.
"""
assert isinstance(n.args[0], Node)
if n.args[0].type == Dyn and isinstance(n.type, TensorType):
n.args[0].type = expand_to_tensor_dim(n.args[0].type, len(n.type.__args__))
if isinstance(n.args[0].type, TensorType):
output = maxpool2d_check(n.args[0].type, module_instance)
n.type = get_greatest_upper_bound(output, n.type)
return n.type
def run_node(self, n: Node) -> Any:
result = super().run_node(n)
if isinstance(result, torch.Tensor):
n.meta['shape'] = result.shape
n.meta['dtype'] = result.dtype
n.meta['stride'] = result.stride()
n.meta['is_quantized'] = result.is_quantized
if n.meta['is_quantized']:
n.meta['qscheme'] = result.qscheme()
if n.meta['qscheme'] in {
torch.per_tensor_affine, torch.per_tensor_symmetric
}:
n.meta['q_scale'] = result.q_scale()
n.meta['q_zero_point'] = result.q_zero_point()
return result