def adjust_order(G, reshape_weights=True, postprocess=True, debug_function=None, steps=None, single_step=False): if steps is None: opts = {'reshape_weights': reshape_weights} selector = AdjusterBase.get_all_handlers(opts) LOG.info("adding transposes to correct tensor order for AT kernels") ConstantInputParameters.clear_compression_state(G) for node in G.nodes(node_classes=tuple(selector)): adjusters = selector[node.__class__] for adjuster, attrs in adjusters: if attrs: not_selected = False for attr, val in attrs.items(): if not hasattr(node, attr): not_selected = True break if callable(val): if not val(getattr(node, attr)): not_selected = True break elif getattr(node, attr) != val: not_selected = True break if not_selected: continue adjuster.adjust(G, node) break add_dimensions(G) if debug_function: debug_function(G) if steps is not None or postprocess: eliminate_transposes(G, debug_function=debug_function, steps=steps, single_step=single_step)
def _common(cls, node, copy_qtype=False, quantized_args=None, **kwargs): all_nodes = kwargs['all_nodes'] valid_name = kwargs['valid_name'] G = kwargs['G'] constant_operation = kwargs.get('constant_operation') constant_int_operation = kwargs.get('constant_int_operation') inputs = [all_nodes[inp] for inp in node.input] if quantized_args: args = [inputs[quantized_args[0][0]], inputs[quantized_args[1][0]]] inp_qtypes = [ cls.get_qtype(inputs[quantized_args[0][1]], inputs[quantized_args[0][2]]), cls.get_qtype(inputs[quantized_args[1][1]], inputs[quantized_args[1][2]]) ] out_qtype = cls.get_qtype(inputs[quantized_args[2][0]], inputs[quantized_args[2][1]]) else: args = inputs assert len(args) == 2 out_qtype = None if all(cls.is_constant(inp) for inp in args) and constant_operation: values = [cls.get_constant(inp) for inp in args] if quantized_args: values = [inp_qtype.dequantize(val) for inp_qtype, val in zip(inp_qtypes, values)] outputs = cls.implied_broadcast(inputs) if constant_int_operation and all(np.issubdtype(val.dtype, np.integer) for val in values): res = constant_int_operation(*values) else: res = constant_operation(*values) if quantized_args: res = out_qtype.quantize(res) if res.size < 10: logger.info("reducing %s to a constant %s", valid_name, res) else: logger.info("reducing %s to a constant", valid_name) params = ConstantInputParameters(valid_name, value=res, dims=Dim.unnamed(outputs[0].known_shape), qtype=out_qtype) else: params_args = kwargs.get('params_args', {}) params = kwargs['params_class'](valid_name, **params_args) outputs = cls.implied_broadcast(inputs) shapes = [] for idx, inp in enumerate(args): G.add_edge(NNEdge(from_node=inp[0], to_node=params, from_idx=inp[1], to_idx=idx)) shapes.append(inp[2].known_shape) if isinstance(params, Broadcastable): params.set_broadcast(shapes) if quantized_args: for qtype, inp in zip(inp_qtypes, args): if cls.is_constant(inp): inp[0].qtype = qtype qrecs = kwargs['qrecs'] qrecs[NodeId(params)] = QRec.scaled(in_qs=inp_qtypes, out_qs=[out_qtype]) if copy_qtype: out_qtype = inputs[0][3] if inputs[0][3] is not None else inputs[1][3] all_nodes[node.output[0]] = (params, 0, outputs[0], out_qtype) return params
def _common(cls, node, **kwargs): all_nodes = kwargs['all_nodes'] G = kwargs['G'] opts = kwargs['opts'] node_opts = kwargs.get("node_opts", None) params_args = kwargs.get('params_args', {}) constant_operation = kwargs.get('constant_operation') inputs = [all_nodes[inp] for inp in node.input] assert len(inputs) == 2 if all(cls.is_constant(inp) for inp in inputs) and constant_operation: LOG.info("reducing %s to a constant", node.name) values = [cls.get_constant(inp) for inp in inputs] output_shapes = cls.implied_broadcast(inputs) params = ConstantInputParameters(node.name, value=constant_operation(*values), dims=Dim.unnamed( output_shapes[0].known_shape)) else: params = kwargs['params_class'](node.name, **params_args) output_shapes = cls.implied_broadcast(inputs) shapes = [] for idx, inp in enumerate(inputs): G.add_edge( NNEdge(from_node=inp[0], to_node=params, from_idx=inp[1], to_idx=idx)) shapes.append(inp[2].known_shape) if isinstance(params, Broadcastable): for idx, shape in enumerate(shapes.copy()): len_diff = len(shape) - len(output_shapes[0].known_shape) if len_diff > 0: if not all(dim is None or dim == 1 for dim in shape[:len_diff:]): in_shapes = ",".join( str(shape) for shape in shapes) raise ValueError( f'strange broadcast {in_shapes} -> {output_shapes[0].shape}' ) shapes[idx] = shape[len_diff::] params.set_broadcast(shapes) if opts.get('load_quantization'): G.quantization[NodeId(params)] = cls.load_tf_quantization( node.input, node.output) if node_opts is not None: params = cls.fuse_activation(node_opts, node.name, params, **kwargs) all_nodes[node.output[0]] = (params, 0, output_shapes[0]) return params
def _common(cls, node, **kwargs): all_nodes = kwargs['all_nodes'] valid_name = kwargs['valid_name'] value = numpy_helper.to_array(node.attrs['value']) params = ConstantInputParameters(valid_name, dims=Dim.unnamed(value.shape), value=value) all_nodes[node.output[0]] = (params, 0, ProvisionalDim(value.shape)) return params
def _common(cls, node, starts, ends, axes, steps, **kwargs): all_nodes = kwargs['all_nodes'] G = kwargs['G'] valid_name = kwargs['valid_name'] x = all_nodes[node.input[0]] x_shape = np.array(x[2].shape) x_rank = len(x_shape) axes = cls._resolve_negative_ranks( axes, len(x_shape)) if axes else tuple(range(x_rank)) axes_rank = len(axes) steps = steps if steps else [1] * axes_rank slices = np.stack([starts, ends, steps]).transpose((1, 0)) p_slices = [] p_shape = [] for idx, dim in enumerate(x_shape): try: if dim is None: p_slices.append(None) p_shape.append(None) else: slice_idx = axes.index(idx) begin, end, step = slices[slice_idx] begin = max(min(begin if begin >= 0 else dim + begin, dim), 0) end = max(min(end if end >= 0 else dim + end, dim), -1) # -sys.maxsize is used to indicate 0 in the reverse slice direction # this makes it compatible with the numpy slice p_slices.append( (begin, -sys.maxsize if end == -1 else end, step)) if step < 0: p_shape.append((begin - end) // -step) else: p_shape.append((end - begin) // step) except ValueError: p_slices.append((0, dim, 1)) p_shape.append(dim) slices = cls._get_real_dim(p_slices) shape = cls._get_real_dim(p_shape) params = StridedSliceParameters(valid_name, act_slice=slices, out_shape=shape) if cls.is_constant(x): x_val = cls.get_constant(x) x_val = params.numpy_slice(x_val) if x_val.size < 10: logger.info("reducing %s to a constant %s", valid_name, x_val) else: logger.info("reducing %s to a constant", valid_name) params = ConstantInputParameters(valid_name, dims=Dim.unnamed(x_val.shape), value=x_val, constant_store=G.constant_store) else: G.add_edge( NNEdge(from_node=x[0], to_node=params, from_idx=x[1], to_idx=0)) all_nodes[node.output[0]] = (params, 0, ProvisionalDim(p_shape)) return params
def _common(cls, node: TFLiteNode, **kwargs): node_opts = node.get_options(CastOptions) G = kwargs['G'] all_nodes = kwargs['all_nodes'] inputs = [all_nodes[t] for t in node.input] x = inputs[0] if node_opts: in_dtype = TFLiteTensorWrapper.TF_TO_NUMPY_TYPE[ node_opts.InDataType()] out_dtype = TFLiteTensorWrapper.TF_TO_NUMPY_TYPE[ node_opts.OutDataType()] else: in_dtype = out_dtype = None if cls.is_constant(x): LOG.info("reducing %s to a constant", node.name) val = cls.get_constant(x) if out_dtype: val = val.astype(out_dtype) params = ConstantInputParameters(node.name, value=val) else: params = CastParameters(node.name, in_dtype=in_dtype, out_dtype=out_dtype) G.add_edge( NNEdge(from_node=x[0], to_node=params, from_idx=x[1], to_idx=0)) all_nodes[node.output[0]] = (params, 0, deepcopy(x[2])) return params
def _common(cls, node, **kwargs): all_nodes = kwargs['all_nodes'] G = kwargs['G'] valid_name = kwargs['valid_name'] inputs = [all_nodes[inp] for inp in node.input] x = inputs[0] x_shape = x[2].shape out_rank = len(x_shape) + len(kwargs['axes']) axes = cls._resolve_negative_ranks(kwargs['axes'], out_rank) old_shape = x_shape.copy() new_shape = [ 1 if new_idx in axes else old_shape.pop(0) for new_idx in range(out_rank) ] pshape = ProvisionalDim(new_shape) if cls.is_constant(x): x_val = cls.get_constant(x) logger.info( f"reducing {valid_name} to a constant {cls.print_small(x_val)}" ) params = ConstantInputParameters(valid_name, value=x_val.reshape(new_shape)) else: old_shape = cls._get_real_dim(x_shape) shape = cls._get_real_dim(new_shape) params = ReshapeParameters(valid_name, old_shape=old_shape, shape=shape) G.add_edge( NNEdge(from_node=x[0], to_node=params, from_idx=x[1], to_idx=0)) all_nodes[node.output[0]] = (params, 0, pshape, x[3]) return params
def _common(cls, node, v13=False, **kwargs): all_nodes = kwargs['all_nodes'] valid_name = kwargs['valid_name'] G = kwargs['G'] inputs = [all_nodes[inp] for inp in node.input] axis = node.attrs.get('axis', None) # may have more than one input i.e. clip x = inputs[0] x_shape = x[2].shape if axis and axis < 0: axis += len(x_shape) axis = cls._trim_axis(axis, x_shape) if axis != 0 and not v13: ValueError( 'LogSoftmax does not support ONNX version < 13 with axis not first' ) if cls.is_constant(x): logger.info("reducing %s to a constant", valid_name) params = ConstantInputParameters( valid_name, value=np.log(softmax_func(cls.get_constant(x), axis=axis))) else: softmax_params = SoftMaxParameters(f'{valid_name}_softmax', axis=axis) G.add_edge( NNEdge(from_node=x[0], to_node=softmax_params, from_idx=x[1], to_idx=0)) params = LogOpParameters(f'{valid_name}_log') G.add_edge(NNEdge(from_node=softmax_params, to_node=params)) all_nodes[node.output[0]] = (params, 0, copy.deepcopy(x[2]), None) return params
def add_constants(G, sub_g): """ adds scalar constants to the subgraphs. If a constant is used in more than one place then it is duplicated """ for node in sub_g.nodes(): for edge in G.in_edges(node.name): if not isinstance(edge.from_node, ConstantInputParameters ) or edge.from_node.out_dims[0].size() > 1: continue const_node = edge.from_node out_edges = G.out_edges(const_node.name) # if constant is connected to more than one node then duplicate it if len(out_edges) > 1: new_const = ConstantInputParameters( G.unique_name(f'{const_node}_dup'), value=const_node.value.copy(), dims=const_node.dims.clone()) G.remove_edge(edge) G.add_edge( NNEdge(from_node=new_const, to_node=edge.to_node, to_idx=edge.to_idx)) sub_g.add_edge( NNEdge(from_node=new_const, to_node=edge.to_node, to_idx=edge.to_idx)) else: sub_g.add_edge( NNEdge(from_node=const_node, to_node=edge.to_node, to_idx=edge.to_idx))
def gen_concat(cls, node, inputs, axis, **kwargs): all_nodes = kwargs['all_nodes'] G = kwargs['G'] valid_name = kwargs['valid_name'] inputs = [all_nodes[inp] for inp in node.input] input_shapes = [inp[2].shape for inp in inputs] axis_sum = sum(shape[axis] for shape in input_shapes) axis = axis if axis >= 0 else len(input_shapes[0]) + axis output_shape = [ axis_sum if idx == axis else dim for idx, dim in enumerate(input_shapes[0]) ] pout_dim = ProvisionalDim(output_shape) none_dims = sum( [1 if dim is None else 0 for dim in output_shape[:axis:]]) if all(cls.is_constant(inp) for inp in inputs): value = np.concatenate([cls.get_constant(inp) for inp in inputs], axis=axis) logger.info( f"reducing {valid_name} to a constant {print_small(value)}") params = ConstantInputParameters(valid_name, value=value) else: params = ConcatParameters(valid_name, axis=axis - none_dims) for idx, inp in enumerate(inputs): G.add_edge( NNEdge(from_node=inp[0], to_node=params, from_idx=inp[1], to_idx=idx)) all_nodes[node.output[0]] = (params, 0, pout_dim, inputs[0][3]) return params
def _common(cls, node: TFLiteNode, **kwargs): all_nodes = kwargs['all_nodes'] G = kwargs['G'] inputs = [all_nodes[t] for t in node.input] x = inputs[0] x_shape = x[2].shape if len(x_shape) != 1: raise ValueError(f'FILL {node.name} expecting 1D tensor for shape') shape = list(cls._verify_constant(inputs[0])) if cls._is_constant(inputs[1]): val = cls._get_constant(inputs[1]) params = ConstantInputParameters(node.name, dims=Dim.unnamed(shape), value=np.full(shape, val), constant_store=G.constant_store) all_nodes[node.output[0]] = (params, 0, ProvisionalDim(shape)) return params else: raise ValueError( f'FILL {node.name} non constant fill values are not currently supported' )
def match(self, G: GraphView, set_identity: bool = True): has_modified = False for node in G.nodes(node_classes=ConstantInputParameters): out_edges = G.out_edges(node.name) if len(out_edges) <= 1: continue has_modified = True LOG.info( 'node %s has more than one out edge and will be duplicated', node.name) idx = 1 for out_edge in out_edges[1::]: new_constant = ConstantInputParameters(f'{node.name}_{idx}', dims=Dim.unnamed( node.dims.shape), value=node.value.copy()) G.remove_edge(out_edge) G.add_edge( NNEdge(from_node=new_constant, to_node=out_edge.to_node, to_idx=out_edge.to_idx)) idx += 1 if set_identity: self.set_identity(G) return has_modified
def common_quantize(cls, in_qtype, out_qtype, node, **kwargs): all_nodes = kwargs['all_nodes'] opts = kwargs['opts'] G = kwargs['G'] inputs = [all_nodes[t] for t in node.input] x = inputs[0] if cls.is_constant(x): LOG.info("reducing %s to a constant", node.name) if out_qtype: val = x[0].value_as(out_qtype) else: val = cls.get_constant(x) params = ConstantInputParameters(node.name, value=val, dims=Dim.unnamed(val.shape), qtype=out_qtype, constant_store=G.constant_store) if opts.get('load_quantization'): G.quantization[NodeId(params)] = MultQuantizationRecord( in_qs=[out_qtype], out_qs=[out_qtype]) else: params = QuantizeParameters(node.name, from_qtype=in_qtype, to_qtype=out_qtype) G.add_edge(NNEdge(from_node=x[0], to_node=params, from_idx=x[1], to_idx=0)) if opts.get('load_quantization'): G.quantization[NodeId(params)] = MultQuantizationRecord( in_qs=[in_qtype], out_qs=[out_qtype]) all_nodes[node.output[0]] = (params, 0, deepcopy(x[2])) return params
def _common(cls, node, **kwargs): all_nodes = kwargs['all_nodes'] G = kwargs['G'] valid_name = kwargs['valid_name'] inputs = [all_nodes[inp] for inp in node.input] if not all(cls.is_constant(inp) for inp in inputs): raise NotImplementedError( "nntool does not support import of graphs with evaluated loops" ) importer = kwargs['importer'] sub_G = NNGraph() all_nodes_clone = all_nodes.copy() importer.import_subgraph(sub_G, node.attrs['body'], {}, all_nodes=all_nodes_clone) if not all( isinstance(inp, (InputParameters, ConstantInputParameters)) for inp in sub_G.inputs()): raise NotImplementedError( "nntool does not support import of graphs with evaluated loops" ) sub_G.add_dimensions() for idx, inp in enumerate(sub_G.inputs()): inp.index = idx logger.info(f"reducing loop {valid_name} to a constant") count = inputs[0][0].value keep_going = inputs[1][0].value loop_carried = [inp[0].value for inp in inputs[2:]] outputs = [np.array([])] * len(node.output) while keep_going and count > 0: executer = GraphExecuter(sub_G) output_tensors = executer.execute([count, keep_going] + loop_carried, silent=True) outp_vals = [ output_tensors[node.step_idx][0] for node in sub_G.outputs() if not isinstance(node, InputParameters) ] keep_going = outp_vals[0] for idx, val in enumerate(outp_vals[1:]): if idx < len(loop_carried): loop_carried[idx] = outputs[idx] = val elif outputs[idx] is None: outputs[idx] = val else: outputs[idx] = np.concatenate((outputs[idx], val)) count -= 1 for idx, outp in enumerate(node.output): params = ConstantInputParameters( G.unique_name(f'{valid_name}_out{idx}'), value=outputs[idx], dims=Dim.unnamed(outputs[idx].shape)) all_nodes[outp] = (params, 0, ProvisionalDim(outputs[idx].shape), None) return None
def gen_dct_matrix(self): norm_factor = np.ones((self.n_dct, self.n_dct)) if self.dct_norm == "ortho" and self.dct_type == 2: norm_factor *= np.sqrt(1 / (2 * self.n_dct)) norm_factor[0] = np.sqrt(1 / (4 * self.n_dct)) if self.dct_norm == "ortho" and self.dct_type == 3: norm_factor *= np.sqrt(1 / (2 * self.n_dct)) norm_factor[0] = np.sqrt(1 / (self.n_dct)) DCT_Coeff = np.zeros((self.n_dct, self.n_dct)) for k in range(self.n_dct): for i in range(self.n_dct): if self.dct_type == 2: coeff = 2 * np.cos(np.pi / (2 * self.n_dct) * k * (2 * i + 1)) elif self.dct_type == 3: coeff = 1 if i == 0 else 2 * np.cos(np.pi / (2 * self.n_dct) * i * (2 * k + 1)) else: raise NotImplementedError( "DCT type 2 and 3 only supported") DCT_Coeff[k, i] = coeff return ConstantInputParameters(self.name + "_DCT_Matrix", value=DCT_Coeff * norm_factor, dims=Dim.unnamed( [self.n_dct, self.n_dct]))
def _common(cls, node, **kwargs): all_nodes = kwargs['all_nodes'] G = kwargs['G'] valid_name = kwargs['valid_name'] inputs = [all_nodes[inp] for inp in node.input] x = inputs[0] x_shape = x[2].shape to_dtype = node.attrs['to'] if cls.is_constant(x): x_val = cls.get_constant(x) x_val = x_val.astype(to_dtype) if x_val.size < 10: logger.info("reducing %s to a constant %s", valid_name, x_val) else: logger.info("reducing %s to a constant", valid_name) params = ConstantInputParameters(valid_name, dims=Dim.unnamed(x_val.shape), value=x_val) else: params = QuantizeParameters(valid_name, to_qtype=QType(dtype=to_dtype)) G.add_edge( NNEdge(from_node=x[0], to_node=params, from_idx=x[1], to_idx=0)) all_nodes[node.output[0]] = (params, 0, ProvisionalDim(x_shape), None) return params
def _common(cls, node, **kwargs): all_nodes = kwargs['all_nodes'] G = kwargs['G'] valid_name = kwargs['valid_name'] if node.attrs.get('value'): value = numpy_helper.to_array(node.attrs['value']) elif node.attrs.get('value_float'): value = np.atleast_1d(node.attrs['value_float'], dtype=np.float32) elif node.attrs.get('value_floats'): value = np.array(node.attrs['value_floats'], dtype=np.float32) elif node.attrs.get('value_int'): value = np.atleast_1d(node.attrs['value_int'], dtype=np.int32) elif node.attrs.get('value_ints'): value = np.array(node.attrs['value_ints'], dtype=np.int32) elif node.attrs.get('value_string') or node.attrs.get('value_strings'): raise NotImplementedError( 'NNTOOL does not support string constants') elif node.attrs.get('sparse_value'): raise NotImplementedError( 'NNTOOL does not support sparse constants') else: raise ValueError('ONNX constant has no value') params = ConstantInputParameters(valid_name, dims=Dim.unnamed(value.shape), value=value) all_nodes[node.output[0]] = (params, 0, ProvisionalDim(value.shape), None) return params
def _common(cls, node, **kwargs): all_nodes = kwargs['all_nodes'] G = kwargs['G'] valid_name = kwargs['valid_name'] inputs = [all_nodes[inp] for inp in node.input] x = inputs[0] x_shape = x[2].shape axes = cls._resolve_negative_ranks(kwargs['axes'], len(x_shape)) if axes: if any(x_shape[axis] != 1 for axis in axes): raise ValueError("axis parameter in node %s is invalid %s" % (valid_name, axes)) new_shape = [dim for idx, dim in enumerate(x_shape) if idx not in axes] else: new_shape = [dim for dim in x_shape if dim != 1] pshape = ProvisionalDim(new_shape) if cls.is_constant(x): logger.info("reducing %s to a constant", valid_name) x_val = cls.get_constant(x) params = ConstantInputParameters(valid_name, value=x_val.reshape(new_shape), constant_store=G.constant_store) else: old_shape = cls._get_real_dim(x_shape) shape = cls._get_real_dim(new_shape) params = ReshapeParameters(valid_name, old_shape=old_shape, shape=shape) G.add_edge(NNEdge(from_node=x[0], to_node=params, from_idx=x[1], to_idx=0)) all_nodes[node.output[0]] = (params, 0, pshape) return params
def _common(cls, node, **kwargs): all_nodes = kwargs['all_nodes'] G = kwargs['G'] valid_name = kwargs['valid_name'] inputs = [all_nodes[inp] for inp in node.input] x = inputs[0] x_shape = x[2].shape axes = cls._resolve_negative_ranks(kwargs['axes'], len(x_shape)) if len(x_shape) == 0: assert len(axes) == 1 and axes[0] == 0 new_shape = [1] else: new_shape = [ item for sublist in [[1, dim] if idx in axes else [dim] for idx, dim in enumerate(x_shape)] for item in sublist ] pshape = ProvisionalDim(new_shape) if cls.is_constant(x): logger.info("reducing %s to a constant", valid_name) x_val = cls.get_constant(x) params = ConstantInputParameters(valid_name, value=x_val.reshape(new_shape)) else: old_shape = cls._get_real_dim(x_shape) shape = cls._get_real_dim(new_shape) params = ReshapeParameters(valid_name, old_shape=old_shape, shape=shape) G.add_edge( NNEdge(from_node=x[0], to_node=params, from_idx=x[1], to_idx=0)) all_nodes[node.output[0]] = (params, 0, pshape) return params
def _common(cls, node, **kwargs): all_nodes = kwargs['all_nodes'] G = kwargs['G'] valid_name = kwargs['valid_name'] inputs = [all_nodes[inp] for inp in node.input] x = inputs[0] x_shape = x[2].shape transpose = node.attrs.get('perm', list(range(len(x_shape) - 1, -1, -1))) transpose = tuple(transpose) pout_shape = [x_shape[i] for i in transpose] new_axes = {} for idx, dim in enumerate(x_shape): if dim is not None: new_axes[idx] = len(new_axes) if cls.is_constant(x): logger.info("reducing %s to a constant", valid_name) x_val = cls.get_constant(x) params = ConstantInputParameters(valid_name, value=x_val.transpose(transpose)) else: transpose = [new_axes[axis] for axis in transpose if x_shape[axis] is not None] if transpose == sorted(transpose): params = NoOPParameters(valid_name, desc="transpose does nothing") else: params = TransposeParameters(valid_name, transpose=transpose) G.add_edge(NNEdge(from_node=x[0], to_node=params, from_idx=x[1], to_idx=0)) all_nodes[node.output[0]] = (params, 0, ProvisionalDim(pout_shape)) return params
def _import_as_matmul(cls, node, inputs, x, y, real_x_shape, real_y_shape, trans_a, trans_b, alpha, beta, **kwargs): G = kwargs['G'] valid_name = kwargs['valid_name'] all_nodes = kwargs['all_nodes'] if trans_a: tparams = TransposeParameters(G.unique_name( f'{valid_name}_tinx'), transpose=(1, 0)) G.add_edge(NNEdge(from_node=x[0], to_node=tparams, from_idx=x[1])) x = (tparams, 0) if trans_b: tparams = TransposeParameters(G.unique_name( f'{valid_name}_tiny'), transpose=(1, 0)) G.add_edge(NNEdge(from_node=y[0], to_node=tparams, from_idx=y[1])) y = (tparams, 0) params = MatMulOpParameters(G.unique_name(valid_name)) G.add_edge( NNEdge(from_node=x[0], to_node=params, from_idx=x[1], to_idx=0)) G.add_edge( NNEdge(from_node=y[0], to_node=params, from_idx=y[1], to_idx=1)) out_dims = params.get_output_size( [Dim.unnamed(real_x_shape), Dim.unnamed(real_y_shape)]) biases = cls.get_constant(inputs[2]) if len(inputs) > 2 else np.zeros(out_dims[0].shape[1]) biases_params = ConstantInputParameters( G.unique_name(f'{valid_name}_biases'), dims=Dim.unnamed(biases.shape), value=biases) G.add_edge( NNEdge(from_node=biases_params, to_node=params, to_idx=2)) cls.record_constant_qrec(inputs[2], biases_params, **kwargs) all_nodes[node.output[0]] = (params, 0, out_dims[0], None) return params
def do_fusions(self, args): """ Carry out the default set of fusions on the graph""" if args.list: table = texttable.Texttable() table.set_cols_align(['l', 'l']) table.set_max_width(120) table.add_rows([['Name', 'Description']] + get_fusions()) self.ppaged(table.draw()) return self._check_graph() state = ConstantInputParameters.save_compression_state(self.G) try: if args.apply: fusions = [get_fusion(name) for name in args.apply] invalid_names = [ args.apply[idx] for idx, fusion in enumerate(fusions) if fusion is None ] if invalid_names: self.perror( f'fusion{"s" if len(invalid_names) > 1 else ""} {", ".join(invalid_names)} not found' ) return elif args.pow2: fusions = [get_pow2_match_group()] elif args.scale8: fusions = [get_scale8_match_group()] else: self.perror( "No fusion set selected. Nothing to do. Select --pow2 or --scale8." ) return for fusion in fusions: fusion.match(self.G) self.G.add_dimensions() if self.G.quantization and verify_quantization(self.G): quantizer = NewQuantizer(self.G) quantizer.quantize() problems = verify_quantization(self.G) if problems: self.perror('quantization issue after fusions') for problem in problems: self.perror(problem) finally: ConstantInputParameters.restore_compression_state(self.G, state)
def rescale_constant(cls, node: ConstantInputParameters, scale, qrecs, dtype=None): qrec = qrecs[NodeId(node)] qtype = qrec.out_qs[0] if (qtype.scale == scale.astype(qtype.scale.dtype) and (dtype is None or dtype == qtype.dtype)): return if node.qtype: node.value = node.dqvalue node.qtype = None qtype.scale = scale if dtype: qtype.dtype = dtype
def _get_initializers(self, G, initializer): return { init.name: (ConstantInputParameters( f'constant_{self._validate_name(init.name)}', dims=Dim.unnamed(init.dims or [1]), value=self._get_numpy_array(init), constant_store=G.constant_store), 0, ProvisionalDim(init.dims)) for init in initializer }
def _common(cls, node, mode='constant', pads=None, constant_value=0, **kwargs): all_nodes = kwargs['all_nodes'] G = kwargs['G'] valid_name = kwargs['valid_name'] inputs = [all_nodes[inp] for inp in node.input] x = inputs[0] x_shape = x[2].shape apads = np.array(pads).reshape((-1, 2)) if cls.is_constant(x): logger.info("reducing %s to a constant", valid_name) val = cls.get_constant(x) if mode == 'constant': val = np.pad(val, apads, mode=mode, constant_values=constant_value) else: val = np.pad(val, apads, mode=mode) params = ConstantInputParameters(valid_name, value=val, constant_store=G.constant_store) all_nodes[node.output[0]] = (params, 0, ProvisionalDim(x_shape)) return params if mode != 'constant': raise ValueError('%s - pad mode %s is not supported' % (valid_name, mode)) if constant_value != 0: raise ValueError('%s - only zero padding is supported' % valid_name) trimmed_pads = tuple( [pad for idx, pad in enumerate(apads) if x_shape[idx] is not None]) if all(sum(trimmed_pad) == 0 for trimmed_pad in trimmed_pads): params = NoOPParameters(valid_name, desc="eliminated pad of 0") pshape = x_shape else: pshape = [ dim + sum(apads[idx]) if dim is not None else None for idx, dim in enumerate(x_shape) ] # pshape = [None if dim is None else dim + sum(apads[idx]) for idx, dim in enumerate(x_shape)] padvals = [(constant_value, constant_value)] * len(trimmed_pads) params = PadParameters(valid_name, padding=trimmed_pads, pad_vals=padvals) G.add_edge( NNEdge(from_node=x[0], to_node=params, from_idx=x[1], to_idx=0)) all_nodes[node.output[0]] = (params, 0, ProvisionalDim(pshape)) return params
def _get_initializers(self, initializer): return { init.name: (ConstantInputParameters(self._validate_name(init.name), dims=Dim.unnamed(init.dims or [1]), value=self._get_numpy_array(init)), 0, Dim.unnamed(init.dims)) for init in initializer }
def _get_initializers(self, G, initializer): return { init.name: (ConstantInputParameters( f'constant_{self._validate_name(init.name)}', dims=Dim.unnamed(init.dims or [1]), value=self._get_numpy_array(init), imported_dtype=self.get_onnx_tensor_dtype(init)), 0, ProvisionalDim(init.dims), None) for init in initializer }
def _common(cls, node, **kwargs): all_nodes = kwargs['all_nodes'] G = kwargs['G'] valid_name = kwargs['valid_name'] inputs = [all_nodes[inp] for inp in node.input] if cls.SINCE_VERSION == 1: shape = np.array(node.attrs["shape"]) else: # since_version >= 5 shape = cls.get_constant(inputs[1]) input_shape = np.array(inputs[0][2].shape) shape = [ dim if dim != 0 else input_shape[idx] for idx, dim in enumerate(shape) ] if -1 in shape: wild_index = shape.index(-1) in_size = prod([1 if dim is None else dim for dim in input_shape]) shape_size = prod( [1 if dim is None or dim <= 0 else dim for dim in shape]) if in_size % shape_size != 0: raise ValueError('invalid reshape') shape[wild_index] = in_size // shape_size shape = np.array(shape) if cls.is_constant(inputs[0]): logger.info("reducing %s to a constant", valid_name) params = ConstantInputParameters(valid_name, value=cls.get_constant( inputs[0]).reshape(shape), dims=Dim.unnamed(shape), constant_store=G.constant_store) pshape = ProvisionalDim(shape) all_nodes[node.output[0]] = (params, 0, pshape) return params # TODO - There must be a better way of doing this # This hacks around the fact that the batch dimension will be in the reshape if input_shape[0] is None and shape[0] == 1: shape = np.array([None] + list(shape[1::])) pshape = ProvisionalDim(shape) # pylint: disable=singleton-comparison old_shape = Dim.unnamed(list(input_shape[input_shape != None])) shape = Dim.unnamed(list(shape[shape != None])) params = ReshapeParameters(valid_name, old_shape=old_shape, shape=shape) inp = inputs[0] G.add_edge( NNEdge(from_node=inp[0], to_node=params, from_idx=inp[1], to_idx=0)) all_nodes[node.output[0]] = (params, 0, pshape) return params
def _common(cls, node: TFLiteNode, **kwargs): node_opts = node.get_options(ConcatenationOptions) G = kwargs['G'] opts = kwargs['opts'] all_nodes = kwargs['all_nodes'] inputs = [all_nodes[t] for t in node.input] inp_shapes = [input[2].shape for input in inputs] buffer_idxes = [tensor.buffer_idx for tensor in node.input] non_zero_idxes = [idx for idx in buffer_idxes if idx != 0] if len(set(non_zero_idxes)) != len(non_zero_idxes): raise NotImplementedError( "concats with multiple versions of the same input are not supported. " "This is normally a graph design problem.") axis = node_opts.Axis() if any(inp_shape[axis] is None for inp_shape in inp_shapes): raise ValueError("concat on undefined axis in node %s" % node.name) def red_func(x, y): return y.copy() if x is None else [ (elem if y[idx] is not None and elem is not None else None) if idx != axis else elem + y[axis] for idx, elem in enumerate(x) ] pout_shape = reduce(red_func, inp_shapes) if all(cls.is_constant(inp) for inp in inputs): # cls.remove_none_from_constants(inputs, pout_shape) LOG.info("reducing %s to a constant", node.name) value = np.concatenate([cls.get_constant(inp) for inp in inputs], axis=axis) params = ConstantInputParameters(node.name, value=value, constant_store=G.constant_store) else: axis -= sum(1 if dim is None else 0 for dim in pout_shape[:axis:]) params = ConcatParameters(node.name, axis=axis, axis_hint=None) for idx, inp in enumerate(inputs): inp_node, inp_idx = cls._maybe_insert_reshape( G, inp, inp_shapes[idx], pout_shape) G.add_edge( NNEdge(from_node=inp_node, to_node=params, from_idx=inp_idx, to_idx=idx)) if opts.get('load_quantization'): G.quantization[NodeId(params)] = cls.load_tf_quantization( node.input, node.output) cls.fuse_activation(node_opts, node.name, params, **kwargs) all_nodes[node.output[0]] = (params, 0, ProvisionalDim(pout_shape)) return params
def do_aquant(self, args: argparse.Namespace): """ Attempt to calculate quantization for graph using one or more sample input files.""" self._check_graph() stats_collector = ActivationRangesCollector() # if replaying state file then load the activation stats if they are present opts = get_options_from_args(args) state = ConstantInputParameters.save_compression_state(self.G) try: if self.replaying_history and self.history_stats: astats = self.history_stats else: input_args = self._get_input_args(args) processed_input = False for file_per_input in glob_input_files(args.input_files, self.G.num_inputs): LOG.info("input file %s", file_per_input) processed_input = True data = [ import_data(input_file, **input_args) for input_file in file_per_input ] stats_collector.collect_stats(self.G, data) if not processed_input: self.perror("No input files found") return astats = stats_collector.stats self._record_stats(astats) if args.force_width: opts['bits'] = args.force_width quantizer = NewQuantizer(self.G, reset_all=True) quantizer.schemes.append(args.scheme) quantizer.set_stats(astats, opts) quantizer.quantize() self.G.add_dimensions() LOG.info("Quantization set. Use qshow command to see it.") finally: ConstantInputParameters.restore_compression_state(self.G, state)