def infer_symbolic_relations(self, n: Node): if HAS_SYMPY: n.type = self.convert_to_sympy_symbols(n.type) if n.op == 'call_function': if n.target in _RULES: return _RULES[n.target](n) else: pass if n.op == 'call_module': module_instance = self.traced.get_submodule(n.target) if type(module_instance) in _RULES: return _RULES[type(module_instance)](n, module_instance) else: pass if n.op == 'output': def get_node_type(a): return a.type n.type = torch.fx.node.map_arg(n.args[0], get_node_type) return n.type else: pass else: pass
def add_inference_rule(n: Node): assert isinstance(n.args[0], Node) assert isinstance(n.args[1], Node) t1 = n.args[0].type t2 = n.args[1].type # handle scalar addition if t1 == int and isinstance(t2, TensorType): n.type = t2 return n.type elif t2 == int and isinstance(t1, TensorType): n.type = t1 return n.type (new_t1, new_t2) = broadcast_types(t1, t2) n.args[0].type = new_t1 n.args[1].type = new_t2 if is_consistent(new_t1, new_t2): # we return the more precise type if is_more_precise(new_t1, new_t2): n.type = new_t2 else: n.type = new_t1 return n.type else: raise TypeError(f'Cannot add arguments {n.args[0]} ({ n.args[0].type}) and {n.args[1]} ({ n.args[1].type}) in node {n}.' f' Types should match ')
def refine_node(self, n: Node): """ Returns a list of equality constraints for call_module and call_function nodes. Models the relation between input and output dimensions using constraints in case they are both tensors. All operations used in resnet50 are defined. """ if n.type is None: n.type = Dyn n.type = self.replace_dyn_with_fresh_var(n.type) if n.op == 'call_function': if n.target in _REFINEMENT_RULES: self.constraints += _REFINEMENT_RULES[n.target](n) else: pass if n.op == 'call_module': module_instance = self.traced.get_submodule(n.target) if type(module_instance) in _REFINEMENT_RULES: self.constraints += _REFINEMENT_RULES[type(module_instance)](n) else: pass if n.op == 'output': assert isinstance(n.args[0], Node) n.type = n.args[0].type else: pass
def run_node(self, n : Node) -> Any: try: result = super().run_node(n) except Exception: traceback.print_exc() raise RuntimeError( f"ShapeProp error for: node={n.format_node()} with " f"meta={n.meta}" ) found_tensor = False def extract_tensor_meta(obj): if isinstance(obj, torch.Tensor): nonlocal found_tensor found_tensor = True return _extract_tensor_metadata(obj) else: return obj meta = map_aggregate(result, extract_tensor_meta) if found_tensor: n.meta['tensor_meta'] = meta n.meta['type'] = type(result) return result
def transpose_inference_rule(n: Node): """ We check that dimentions for the transpose operations are within range of the tensor type of the node """ if n.target == torch.transpose: assert isinstance(n.args[0], Node) t = n.args[0].type assert isinstance(n.args[1], int) assert isinstance(n.args[2], int) dim1, dim2 = n.args[1], n.args[2] if t == Dyn: n.type = Dyn return n.type elif isinstance(t, TensorType): if 0 <= dim1 < len(t.__args__) and 0 <= dim2 < len(t.__args__): new_type = list(t.__args__) new_type[dim1], new_type[dim2] = new_type[dim2], new_type[dim1] final = TensorType(new_type) n.type = get_greatest_upper_bound(n.type, final) return n.type else: raise TypeError( f'Cannot transpose {dim1} and {dim2} in type {t} for node {n}' ) else: raise TypeError( f'Cannot transpose {dim1} and {dim2} in type {t} for node {n}')
def transpose_inference_rule(n: Node): if n.target == torch.transpose: assert isinstance(n.args[0], Node) t = n.args[0].type assert isinstance(n.args[1], int) assert isinstance(n.args[2], int) dim1, dim2 = n.args[1], n.args[2] if t == Dyn: n.type = Dyn return n.type elif isinstance(t, TensorType): if 0 <= dim1 < len(t.__args__) and 0 <= dim2 < len(t.__args__): new_type = list(t.__args__) new_type[dim1], new_type[dim2] = new_type[dim2], new_type[dim1] final = TensorType(new_type) n.type = final return n.type else: raise TypeError(f'Cannot transpose {dim1} and {dim2} in type {t} for node {n}') else: raise TypeError(f'Cannot transpose {dim1} and {dim2} in type {t} for node {n}')
def type_check_node(self, n: Node): """ Type check a given fx node. Current operations: - Reshape - Transpose - Add """ if n.op == 'placeholder': if n.type is None: n.type = Dyn else: return n.type if n.op == 'call_function': if n.target in _INFERENCE_RULES: return _INFERENCE_RULES[n.target](n) else: raise RuntimeError(f'No inference rule registered for target {n.target}!') if n.op == 'output': assert isinstance(n.args[0], Node) n.type = n.args[0].type return n.type else: raise NotImplementedError("Method not yet implemented")
def bn2d_inference_rule(n: Node, module_instance): """ Given a BatchNorm2D instance and a node check the following conditions: - the input type can be expanded to a size 4 tensor: t = (x_1, x_2, x_3, x_4) - the current node type can be expanded to a size 4 tensor: t' = (x_1', x_2', x_3', x_4') - t is consistent with t' - x_2 is consistent with the module's num_features - x_2' is consistent with the module's num_features output type: the more precise type of t and t' """ assert isinstance(n.args[0], Node) n.args[0].type = expand_to_tensor_dim(n.args[0].type, 4) arg_type = n.args[0].type n.type = expand_to_tensor_dim(n.type, 4) # we check the conditions on the incoming argument # and any existing annotation # we also check for consistency between both annotations if is_consistent(arg_type.__args__[1], module_instance.num_features) and \ is_consistent(n.type.__args__[1], module_instance.num_features) and \ is_consistent(arg_type, n.type): # we choose the more precise type # to be the node type # so if an incoming argument has more type information # we set this node's type to be the argument type n.type = get_greatest_upper_bound(arg_type, n.type) return n.type else: raise TypeError(f'Cannot apply {module_instance} with input type {arg_type} and existing type {n.type} on {n}')
def run_node(self, n: Node) -> Any: result = super().run_node(n) if isinstance(result, torch.Tensor): n.shape = result.shape # type: ignore n.dtype = result.dtype # type: ignore return result
def type_check_node(self, n: Node): """ Type check a given fx node. Current operations: - Reshape - Transpose - Add - Relu - conv2d - batchnorm2d - flatten - maxpool2d - adaptiveavgpool2d - linear """ if n.type is None: n.type = Dyn if n.op == 'placeholder': return n.type elif n.op == 'get_attr': t = get_parameter(self.traced, n.target) # type: ignore[arg-type] if isinstance(t.data, torch.Tensor): n.type = TensorType(t.data.shape) return n.type elif n.op == 'call_function': if n.target == getattr: assert getattr in _INFERENCE_RULES return _INFERENCE_RULES[n.target](n, self.traced) elif n.target in _INFERENCE_RULES: return _INFERENCE_RULES[n.target](n) else: raise RuntimeError( f'No inference rule registered for target {n.target}!') elif n.op == 'call_module': module_instance = self.traced.get_submodule(n.target) if type(module_instance) in _INFERENCE_RULES: return _INFERENCE_RULES[type(module_instance)](n, module_instance) else: raise RuntimeError( f'No inference rule registered for class {type(module_instance)}!' ) elif n.op == 'output': def get_node_type(a): return a.type n.type = torch.fx.node.map_arg(n.args[0], get_node_type) return n.type else: raise NotImplementedError(f"Method {n.op} not yet implemented")
def run_node(self, n: Node) -> Any: result = super().run_node(n) if isinstance(result, torch.Tensor): n.meta['shape'] = result.shape n.meta['dtype'] = result.dtype n.meta['stride'] = result.stride() n.meta['is_quantized'] = result.is_quantized memory_formats = { torch.contiguous_format, torch.channels_last, torch.channels_last_3d, } memory_format = None for query_format in memory_formats: if result.is_contiguous(memory_format=query_format): memory_format = query_format break n.meta['memory_format'] = memory_format if n.meta['is_quantized']: n.meta['qscheme'] = result.qscheme() if n.meta['qscheme'] in { torch.per_tensor_affine, torch.per_tensor_symmetric }: n.meta['q_scale'] = result.q_scale() n.meta['q_zero_point'] = result.q_zero_point() return result
def flatten_inference_rule(n: Node): """ Applies the flatten shape information to the input then gets the greatest upper bound of the resulting type and the existing type """ assert isinstance(n.args[0], Node) # set the default start and end dims start_dim = 1 end_dim = -1 if len(n.args) > 1: assert isinstance(n.args[1], int) start_dim = n.args[1] if len(n.args) > 2: assert isinstance(n.args[2], int) end_dim = n.args[2] if n.args[0].type == Dyn and isinstance(n.type, TensorType): n.args[0].type = expand_to_tensor_dim(n.args[0].type, len(n.type.__args__)) if isinstance(n.args[0].type, TensorType): output_type = flatten_check(n.args[0].type, start_dim, end_dim) n.type = get_greatest_upper_bound(output_type, n.type) return n.type
def conv2d_inference_rule(n: Node, module_instance): """ Given a Conv2D instance and a node check the following conditions: - the input type can be expanded to a size 4 tensor: t = (x_1, x_2, H, W) - the current node type can be expanded to a size 4 tensor: t' = (x_1', x_2', x_3', x_4') - x_2 is consistent with the module's in_channels - let o = (x_1, out_channels, H_out, W_out) then the output is the greatest upper bound of o and the existing node type t'. """ assert isinstance(n.args[0], Node) n.args[0].type = expand_to_tensor_dim(n.args[0].type, 4) arg_type = n.args[0].type curr_node_type = expand_to_tensor_dim(n.type, 4) if is_consistent(arg_type.__args__[1], module_instance.in_channels): w_in = arg_type.__args__[3] h_in = arg_type.__args__[2] h_out = calculate_out_dimension(h_in, module_instance, 0) w_out = calculate_out_dimension(w_in, module_instance, 1) new_type = TensorType((arg_type.__args__[0], module_instance.out_channels, h_out, w_out)) gub = get_greatest_upper_bound(new_type, curr_node_type) n.type = gub return n.type else: raise TypeError(f'Cannot apply {module_instance} with input type { arg_type} and existing type {n.type} on {n}')
def run_node(self, n: Node) -> Any: result = super().run_node(n) if isinstance(result, torch.Tensor): n.meta['tensor_meta'] = extract_tensor_metadata(result) return result
def _nodes_are_equal(self, pn: Node, gn: Node) -> bool: # TODO: match args and kwargs # if exact match for placeholder is not required, then use placeholder as a wildcard if not self.match_placeholder and pn.op == "placeholder": return True if pn.target == torch.ops.pseudo.any: return True if pn.target == torch.ops.pseudo.oneof: permissible_targets: List[str] = pn.kwargs.get( "targets", list()) # type: ignore[assignment] assert isinstance(permissible_targets, list), \ "pseudo.oneof(permissible_targets=[\"foo\", \"bar\"]) only accept targets as a list" assert len( permissible_targets ) > 0, "please specific as least one target for pseudo.oneof" if gn._pretty_print_target(gn.target) in permissible_targets: return True if pn.op == gn.op: if pn.op == "placeholder" or pn.op == "output": return True return pn.target == gn.target return False
def linear_inference_rule(n: Node, module_instance): assert isinstance(n.args[0], Node) if n.args[0].type == Dyn and isinstance(n.type, TensorType): n.args[0].type = expand_to_tensor_dim(n.args[0].type, len(n.type.__args__)) if isinstance(n.args[0].type, TensorType): output_type = linear_check(n.args[0].type, module_instance) n.type = get_greatest_upper_bound(output_type, n.type) return n.type
def run_node(self, n: Node) -> Any: result = super().run_node(n) found_tensor = False def extract_tensor_meta(obj): if isinstance(obj, torch.Tensor): nonlocal found_tensor found_tensor = True return extract_tensor_metadata(obj) else: return obj meta = map_aggregate(result, extract_tensor_meta) if found_tensor: n.meta['tensor_meta'] = meta n.meta['type'] = type(result) return result
def get_attr_inference_rule(n: Node, traced): attr_node = n.args[0] attr_name = n.args[1] if attr_name == "shape": n.type = Dyn else: raise TypeError("Not yet implelemted") # TODO. We leave it like this till we add a type to represent tensor sizes return n.type
def reshape_inference_rule(n: Node): assert isinstance(n.args[0], Node) t1 = n.args[0].type assert isinstance(n.args[1], list) t2 = n.args[1] t2_type = TensorType([Dyn if elem == -1 else elem for elem in t2]) # if we do not know the original tensor dimension, # we return the required dimension if t1 == Dyn: n.type = t2_type return t2_type # if any of the dimensions are unknown, # we check for divisibility elif isinstance(t1, TensorType) and Dyn in t1.__args__ or -1 in t2: assert isinstance(t1, TensorType) a = [e if e != Dyn else 1 for e in t1.__args__] p1 = reduce(lambda x, y: x * y, a) p2 = reduce(lambda x, y: x * y, t2) if p1 % p2 == 0 or p2 % p1 == 0: n.type = t2_type return t2_type else: raise TypeError( f'Cannot reshape in node {n} from {t1} to {t2_type}') # if all dimensions are known we check the products elif isinstance(t1, TensorType): p1 = reduce(lambda x, y: x * y, t1.__args__) p2 = reduce(lambda x, y: x * y, t2) if p1 == p2: n.type = t2_type return t2_type else: raise TypeError( f'Cannot reshape in node {n} from {t1} to {t2_type}') else: raise TypeError(f'Cannot reshape in node {n} from {t1} to {t2_type}')
def type_check_node(self, n: Node): """ Type check a given fx node. Current operations: - Reshape - Transpose - Add - Relu - conv2d - batchnorm2d - flatten - maxpool2d - adaptiveavgpool2d - linear """ if n.type is None: n.type = Dyn if n.op == 'placeholder': return n.type if n.op == 'call_function': if n.target in _INFERENCE_RULES: return _INFERENCE_RULES[n.target](n) else: raise RuntimeError(f'No inference rule registered for target {n.target}!') if n.op == 'call_module': module_instance = self.traced.get_submodule(n.target) if type(module_instance) in _INFERENCE_RULES: return _INFERENCE_RULES[type(module_instance)](n, module_instance) else: raise RuntimeError(f'No inference rule registered for class {type(module_instance)}!') if n.op == 'output': assert isinstance(n.args[0], Node) n.type = n.args[0].type return n.type else: raise NotImplementedError("Method not yet implemented")
def adaptiveavgpool2d_inference_rule(n: Node, module_instance): """ The input and output sizes should be the same except for the last two dimensions taken from the input, which represent width and height """ assert isinstance(n.args[0], Node) if n.args[0].type == Dyn and isinstance(n.type, TensorType): n.args[0].type = expand_to_tensor_dim(n.args[0].type, len(n.type.__args__)) if isinstance(n.args[0].type, TensorType): output_type = adaptiveavgpool2d_check(n.args[0].type, module_instance) n.type = get_greatest_upper_bound(n.type, output_type) return n.type
def conv_rule(n: Node, module_instance): assert isinstance(n.args[0], Node) arg_type = n.args[0].type if isinstance(arg_type, TensorType) and isinstance(n.type, TensorType): w_in = arg_type.__args__[3] h_in = arg_type.__args__[2] h_out = calculate_out_dimension(h_in, module_instance, 0) w_out = calculate_out_dimension(w_in, module_instance, 1) new_type = TensorType( (n.type.__args__[0], n.type.__args__[1], h_out, w_out)) n.type = new_type return new_type
def relu_inference_rule(n: Node, module_instance): """ Input and output shapes should be equal. """ assert isinstance(n.args[0], Node) if n.args[0].type == Dyn and isinstance(n.type, TensorType): n.args[0].type = expand_to_tensor_dim(n.args[0].type, len(n.type.__args__)) if isinstance(n.args[0].type, TensorType): n.type = get_greatest_upper_bound(n.args[0].type, n.type) return n.type
def add_inference_rule(n: Node): """ Apply the addition inference rule. This includes: - scalar addition - broadcasting semantics Note that we always return the least precise type between the operands (after applying broadcasting) to be the final type of the operation Note that we do not modify the operand types themselves after applying broadcasting to them. We only use them to calculate the final type """ assert isinstance(n.args[0], Node) assert isinstance(n.args[1], Node) t1 = n.args[0].type t2 = n.args[1].type # handle scalar addition if t1 == int and isinstance(t2, TensorType): n.type = t2 return n.type # handle scalar addition elif t2 == int and isinstance(t1, TensorType): n.type = t1 return n.type # we bring the new types to the point where # we can check for consistency # any inconsistency would not have been caused # by broadcasting at this point (new_t1, new_t2) = broadcast_types(t1, t2) if new_t1 != t1 or new_t2 != t2: n.meta['broadcast'] = True n.meta[str(n.args[0])] = new_t1 n.meta[str(n.args[1])] = new_t2 else: n.meta['broadcast'] = False new_t1 = t1 if not n.meta['broadcast'] else new_t1 new_t2 = t2 if not n.meta['broadcast'] else new_t2 # we check for consistency between the new types if is_consistent(new_t1, new_t2): # we return the less precise type because # broadcasting may have happened # for operands with shape [1,2,Dyn] and [1,2,1] # we have to assign the node [1,2,Dyn] if is_more_precise(new_t1, new_t2): n.type = new_t2 else: n.type = new_t1 return n.type else: raise TypeError( f'Cannot add arguments {n.args[0]} ({ n.args[0].type}) and {n.args[1]} ({ n.args[1].type}) in node {n}.' f' Types should match ')
def linear_inference_rule(n: Node, module_instance): """ Applies the shape information to the input then gets the greatest upper bound of the resulting type and the existing type """ assert isinstance(n.args[0], Node) if n.args[0].type == Dyn and isinstance(n.type, TensorType): n.args[0].type = expand_to_tensor_dim(n.args[0].type, len(n.type.__args__)) if isinstance(n.args[0].type, TensorType): output_type = linear_check(n.args[0].type, module_instance) n.type = get_greatest_upper_bound(output_type, n.type) return n.type
def reshape_inference_rule(n: Node): """ Without dynamism, the rule checks that the product of the elements of the argument tensor type is equal to the product of the elements of the required shape. We gradualize this rule by adding a case to handle fully dynamic input as well as input where some of the tensor dimensions are unknown. In this case we check for divisibility """ assert isinstance(n.args[0], Node) t1 = n.args[0].type assert isinstance(n.args[1], list) t2 = n.args[1] t2_type = TensorType([Dyn if elem == -1 else elem for elem in t2]) # if we do not know the original tensor dimension, # we return the required dimension if t1 == Dyn: n.type = t2_type return t2_type # if any of the dimensions are unknown, # we check for divisibility elif isinstance(t1, TensorType): assert isinstance(t1, TensorType) a = [e if e != Dyn else 1 for e in t1.__args__] p1 = reduce(lambda x, y: x * y, a) p2 = reduce(lambda x, y: x * y, t2) if p1 % p2 == 0 or p2 % p1 == 0: n.type = t2_type return t2_type else: raise TypeError( f'Cannot reshape in node {n} from {t1} to {t2_type}') else: raise TypeError(f'Cannot reshape in node {n} from {t1} to {t2_type}')
def conv_rule(n: Node, module_instance): """ Represents the outout in terms of an algrbraic expression w.r.t the input when possible """ assert isinstance(n.args[0], Node) arg_type = n.args[0].type if isinstance(arg_type, TensorType) and isinstance(n.type, TensorType): w_in = arg_type.__args__[3] h_in = arg_type.__args__[2] h_out = calculate_out_dimension(h_in, module_instance, 0) w_out = calculate_out_dimension(w_in, module_instance, 1) new_type = TensorType( (n.type.__args__[0], n.type.__args__[1], h_out, w_out)) n.type = new_type return new_type
def get_attr_inference_rule(n: Node, traced): """ The current getattr rule only handles the shape attribute Can be extended to other attributes The most representitive type we have is "Dyn" but the system can be extended with more types, such as a type to represent shapes """ attr_node = n.args[0] attr_name = n.args[1] if attr_name == "shape": n.type = Dyn else: raise TypeError("Not yet implelemted") # TODO. We leave it like this till we add a type to represent tensor sizes return n.type
def maxpool2d_inference_rule(n: Node, module_instance): """ Given a MaxPool2D instance and a node check the following conditions: - Input size matches size 3 or 4 - Current node type is consistent with the output type we will calculate - Input size matches output size and the last two dimensions of the output are w_out and h_out. The remaining dimensions are the same as the input - Our final result is the greatest upper bound of the output we calculate and the current node type. """ assert isinstance(n.args[0], Node) if n.args[0].type == Dyn and isinstance(n.type, TensorType): n.args[0].type = expand_to_tensor_dim(n.args[0].type, len(n.type.__args__)) if isinstance(n.args[0].type, TensorType): output = maxpool2d_check(n.args[0].type, module_instance) n.type = get_greatest_upper_bound(output, n.type) return n.type
def run_node(self, n: Node) -> Any: result = super().run_node(n) if isinstance(result, torch.Tensor): n.meta['shape'] = result.shape n.meta['dtype'] = result.dtype n.meta['stride'] = result.stride() n.meta['is_quantized'] = result.is_quantized if n.meta['is_quantized']: n.meta['qscheme'] = result.qscheme() if n.meta['qscheme'] in { torch.per_tensor_affine, torch.per_tensor_symmetric }: n.meta['q_scale'] = result.q_scale() n.meta['q_zero_point'] = result.q_zero_point() return result