def inference_cost_conv(model, node, discount_sparsity):
# extract info about the conv kernel attributes
k = get_by_name(node.attribute, "kernel_shape").ints
k_prod = np.prod(k)
group = get_by_name(node.attribute, "group")
if group is None:
group = 1
else:
group = group.i
# extract info from tensor shapes and datatypes
(i_dtype, w_dtype, o_dtype) = get_node_tensor_dtypes(model, node)
(i_shape, w_shape, o_shape) = get_node_tensor_shapes(model, node)
bsize = i_shape[0]
ifm_ch = i_shape[1]
ofm_ch = o_shape[1]
assert ofm_ch == w_shape[0], "Mismatch in output channels"
assert ofm_ch % group == 0, "Invalid group setting: " + str(node)
ofm_pix_total = np.prod(o_shape[2:])
n_macs = bsize * (ofm_ch // group) * ifm_ch * k_prod * ofm_pix_total
w_mem = np.prod(w_shape)
o_mem = np.prod(o_shape)
if discount_sparsity:
wname = node.input[1]
density = get_node_weight_density(model, wname)
n_macs *= density
w_mem *= density
idt_name = i_dtype.name
wdt_name = w_dtype.name
odt_name = o_dtype.name
mac_op_type_str = "op_mac_%s_%s" % (idt_name, wdt_name)
w_mem_type_str = "mem_w_%s" % (wdt_name)
o_mem_type_str = "mem_o_%s" % (odt_name)
ret = {mac_op_type_str: n_macs, w_mem_type_str: w_mem, o_mem_type_str: o_mem}
return ret
def set_tensor_datatype(self, tensor_name, datatype):
"""Sets the FINN DataType of tensor with given name."""
graph = self._model_proto.graph
qnt_annotations = graph.quantization_annotation
ret = util.get_by_name(qnt_annotations, tensor_name, "tensor_name")
if ret is not None:
ret_dt = util.get_by_name(ret.quant_parameter_tensor_names,
"finn_datatype", "key")
if ret_dt is not None:
if datatype is None:
ret_dt.Clear()
else:
ret_dt.value = datatype.name
elif datatype is not None:
dt = onnx.StringStringEntryProto()
dt.key = "finn_datatype"
dt.value = datatype.name
ret.quant_parameter_tensor_names.append(dt)
elif datatype is not None:
qa = onnx.TensorAnnotation()
dt = onnx.StringStringEntryProto()
dt.key = "finn_datatype"
dt.value = datatype.name
qa.tensor_name = tensor_name
qa.quant_parameter_tensor_names.append(dt)
qnt_annotations.append(qa)
def set_tensor_layout(self, tensor_name, data_layout):
"""Sets the data layout annotation of tensor with given name. See
get_tensor_layout for examples."""
tensor_shape = self.get_tensor_shape(tensor_name)
assert type(data_layout) == list, "data_layout must be a list"
if tensor_shape is not None:
assert len(tensor_shape) == len(
data_layout
), """Mismatch between number
of dimensions of tensor shape and data layout annotation."""
graph = self._model_proto.graph
qnt_annotations = graph.quantization_annotation
ret = util.get_by_name(qnt_annotations, tensor_name, "tensor_name")
if ret is not None:
ret_tl = util.get_by_name(
ret.quant_parameter_tensor_names, "tensor_layout", "key"
)
if ret_tl is not None:
ret_tl.value = str(data_layout)
else:
tl = onnx.StringStringEntryProto()
tl.key = "tensor_layout"
tl.value = str(data_layout)
ret.quant_parameter_tensor_names.append(tl)
else:
qa = onnx.TensorAnnotation()
dt = onnx.StringStringEntryProto()
dt.key = "tensor_layout"
dt.value = str(data_layout)
qa.tensor_name = tensor_name
qa.quant_parameter_tensor_names.append(dt)
qnt_annotations.append(qa)
def rename_tensor(self, old_name, new_name):
"""Renames a tensor from old_name to new_name."""
graph = self.graph
# sweep over inputs
if util.get_by_name(graph.input, old_name) is not None:
util.get_by_name(graph.input, old_name).name = new_name
# sweep over outputs
if util.get_by_name(graph.output, old_name) is not None:
util.get_by_name(graph.output, old_name).name = new_name
# sweep over value_info
if util.get_by_name(graph.value_info, old_name) is not None:
util.get_by_name(graph.value_info, old_name).name = new_name
# sweep over initializers
if util.get_by_name(graph.initializer, old_name) is not None:
util.get_by_name(graph.initializer, old_name).name = new_name
# sweep over quantization annotations
if (
util.get_by_name(graph.quantization_annotation, old_name, "tensor_name")
is not None
):
util.get_by_name(
graph.quantization_annotation, old_name, "tensor_name"
).tensor_name = new_name
# sweep over node i/o
for n in graph.node:
if old_name in n.input:
n.input[list(n.input).index(old_name)] = new_name
if old_name in n.output:
n.output[list(n.output).index(old_name)] = new_name
def apply(self, model):
# TODO we currently assume that all dataflow nodes are connected to
# each other, forming a single partition. check the assumption and/or
# improve this.
all_nodes = list(model.graph.node)
df_nodes = filter(
lambda x: get_by_name(x.attribute, "backend") is not None,
all_nodes)
df_nodes = filter(
lambda x: get_by_name(x.attribute, "backend").s.decode("UTF-8") ==
"fpgadataflow",
df_nodes,
)
df_nodes = list(df_nodes)
non_df_nodes = filter(lambda x: x not in df_nodes, all_nodes)
non_df_nodes = list(non_df_nodes)
if len(df_nodes) == 0:
# no changes if no dataflow nodes are present
return (model, False)
else:
# partition the model into two models
df_model = copy.deepcopy(model)
non_df_model = model
# remove all non-dataflow nodes from the dataflow model
for node_to_remove in non_df_nodes:
df_model.graph.node.remove(node_to_remove)
# identify the entry and exit points for the dataflow part
df_in = df_model.graph.node[0].input[0]
df_out = df_model.graph.node[-1].output[0]
df_in_vi = df_model.get_tensor_valueinfo(df_in)
df_out_vi = df_model.get_tensor_valueinfo(df_out)
# set df graph in/out to be df_in/df_out
df_model.graph.input.remove(df_model.graph.input[0])
df_model.graph.input.insert(0, df_in_vi)
df_model.graph.output.remove(df_model.graph.output[0])
df_model.graph.output.insert(0, df_out_vi)
df_model_dir = make_build_dir("dataflow_partition_")
df_model_filename = df_model_dir + "/df_model.onnx"
df_model.save(df_model_filename)
# remove all dataflow nodes from the non-dataflow model
# keep track of where the dataflow part starts
df_start_ind = all_nodes.index(df_nodes[0])
for node_to_remove in df_nodes:
non_df_model.graph.node.remove(node_to_remove)
# create StreamingDataflow node with df_in/df_out io
df_node = helper.make_node(
"StreamingDataflowPartition",
[df_in],
[df_out],
# use the model attribute to mark the df model
model=df_model_filename,
)
non_df_model.graph.node.insert(df_start_ind, df_node)
return (non_df_model, False)
def get_tensor_sparsity(self, tensor_name):
"""Returns the sparsity of a given tensor as dictionary."""
graph = self._model_proto.graph
qnt_annotations = graph.quantization_annotation
ret = util.get_by_name(qnt_annotations, tensor_name, "tensor_name")
if ret is not None:
ret = util.get_by_name(ret.quant_parameter_tensor_names,
"tensor_sparsity", "key")
if ret is not None:
return eval(ret.value)
return None
def get_tensor_datatype(self, tensor_name):
"""Returns the FINN DataType of tensor with given name."""
graph = self._model_proto.graph
qnt_annotations = graph.quantization_annotation
ret = util.get_by_name(qnt_annotations, tensor_name, "tensor_name")
if ret is not None:
ret = util.get_by_name(ret.quant_parameter_tensor_names,
"finn_datatype", "key")
if ret is not None:
return DataType[ret.value]
# TODO maybe use native ONNX tensor type instead of assuming fp32?
return DataType["FLOAT32"]
def set_tensor_shape(self, tensor_name, tensor_shape, dtype=TensorProto.FLOAT):
"""Assigns shape in ValueInfoProto for tensor with given name."""
new_vi = oh.make_tensor_value_info(tensor_name, dtype, tensor_shape)
# find what container tis tensor's ValueInfo lives in
# if not found anywhere, we assume it's a new value_info
target_container = self.graph.value_info
if util.get_by_name(self.graph.input, tensor_name) is not None:
target_container = self.graph.input
if util.get_by_name(self.graph.output, tensor_name) is not None:
target_container = self.graph.output
# remove from target container and add new
util.remove_by_name(target_container, tensor_name)
target_container.append(new_vi)
def apply(self, model):
graph = model.graph
node_ind = 0
graph_modified = False
for n in graph.node:
node_ind += 1
if n.op_type == "Transpose" and not model.is_fork_node(n):
perms = list(get_by_name(n.attribute, "perm").ints)
if perms == [0, 3, 1, 2]:
mt_cand = model.find_consumer(n.output[0])
if mt_cand.op_type == "MultiThreshold" and not model.is_fork_node(
mt_cand
):
final_t_cand = model.find_consumer(mt_cand.output[0])
if final_t_cand.op_type == "Transpose":
perms = list(
get_by_name(final_t_cand.attribute, "perm").ints
)
if perms == [0, 2, 3, 1]:
mt = getCustomOp(mt_cand)
mt.set_nodeattr("data_layout", "NHWC")
# get rid of tranpose nodes, wire MT directly
mt_cand.input[0] = n.input[0]
mt_cand.output[0] = final_t_cand.output[0]
graph.node.remove(n)
graph.node.remove(final_t_cand)
graph_modified = True
else:
mt = getCustomOp(mt_cand)
mt.set_nodeattr("data_layout", "NHWC")
# get rid of first tranpose node
mt_cand.input[0] = n.input[0]
graph.node.remove(n)
# fix output shape for MultiThreshold
mt_ishape = model.get_tensor_shape(mt_cand.input[0])
model.set_tensor_shape(mt_cand.output[0], mt_ishape)
# re-insert Transpose behind MultiThreshold
transpose_output = model.make_new_valueinfo_name()
new_transpose = oh.make_node(
"Transpose",
[mt_cand.output[0]],
[transpose_output],
perm=[0, 3, 1, 2],
)
graph.node.insert(node_ind + 1, new_transpose)
final_t_cand.input[0] = transpose_output
graph_modified = True
if graph_modified:
model = model.transform(InferDataTypes())
return (model, graph_modified)
def apply(self, model):
graph = model.graph
node_ind = 0
graph_modified = False
for n in graph.node:
node_ind += 1
if n.op_type == "Transpose" and not model.is_fork_node(n):
perms = list(get_by_name(n.attribute, "perm").ints)
if perms == [0, 3, 1, 2]:
mt_cand = model.find_consumer(n.output[0])
if mt_cand.op_type == "MultiThreshold" and not model.is_fork_node(
mt_cand):
final_t_cand = model.find_consumer(mt_cand.output[0])
if final_t_cand.op_type == "Transpose":
perms = list(
get_by_name(final_t_cand.attribute,
"perm").ints)
if perms == [0, 2, 3, 1]:
mt = getCustomOp(mt_cand)
mt.set_nodeattr("data_layout", "NHWC")
# get rid of tranpose nodes, wire MT directly
mt_cand.input[0] = n.input[0]
mt_cand.output[0] = final_t_cand.output[0]
graph.node.remove(n)
graph.node.remove(final_t_cand)
graph_modified = True
elif final_t_cand.op_type == "Reshape":
oshape = model.get_tensor_shape(
final_t_cand.output[0])
if len(oshape) == 2:
# transition to FC part, can still use NHWC
mt = getCustomOp(mt_cand)
mt.set_nodeattr("data_layout", "NHWC")
# get rid of first tranpose node
mt_cand.input[0] = n.input[0]
# fix output shape for MultiThreshold
mt_ishape = model.get_tensor_shape(
mt_cand.input[0])
(b, h, w, c) = mt_ishape
assert (h == 1
and w == 1), """Untested spatial dim
in conv->fc transition, proceed with caution!"""
model.set_tensor_shape(mt_cand.output[0],
mt_ishape)
graph.node.remove(n)
graph_modified = True
if graph_modified:
model = model.transform(InferDataTypes())
return (model, graph_modified)
def apply(self, model):
graph = model.graph
node_ind = 0
graph_modified = False
for n in graph.node:
node_ind += 1
if n.op_type == "MaxPool":
consumer = model.find_consumer(n.output[0])
producer = model.find_producer(n.input[0])
if consumer is not None and consumer.op_type == "Transpose":
perms = list(get_by_name(consumer.attribute, "perm").ints)
if perms == [0, 2, 3, 1]:
n.op_type = "MaxPoolNHWC"
n.domain = "finn.custom_op.general"
start_name = n.input[0]
mid_name = consumer.input[0]
end_name = consumer.output[0]
(b, c, hi, wi) = model.get_tensor_shape(start_name)
(b, c, ho, wo) = model.get_tensor_shape(mid_name)
consumer.input[0] = start_name
consumer.output[0] = mid_name
n.input[0] = mid_name
n.output[0] = end_name
model.set_tensor_shape(mid_name, (b, hi, wi, c))
model.set_tensor_shape(end_name, (b, ho, wo, c))
graph.node.remove(consumer)
graph.node.insert(node_ind - 1, consumer)
graph_modified = True
elif producer is not None and producer.op_type == "Transpose":
perms = list(get_by_name(producer.attribute, "perm").ints)
if perms == [0, 3, 1, 2]:
n.op_type = "MaxPoolNHWC"
n.domain = "finn.custom_op.general"
start_name = producer.input[0]
mid_name = n.input[0]
end_name = n.output[0]
(b, hi, wi, c) = model.get_tensor_shape(start_name)
(b, c, ho, wo) = model.get_tensor_shape(end_name)
producer.input[0] = mid_name
producer.output[0] = end_name
n.input[0] = start_name
n.output[0] = mid_name
model.set_tensor_shape(mid_name, (b, ho, wo, c))
model.set_tensor_shape(end_name, (b, c, ho, wo))
graph.node.remove(producer)
graph.node.insert(node_ind, producer)
graph_modified = True
return (model, graph_modified)
def test_change_datalayout_quantavgpool(s, k, ibits, obits, signed, c, idim):
n = 1
odim = compute_pool_output_dim(idim, k, s)
# determine input FINN datatype
if signed is True:
prefix = "INT"
else:
prefix = "UINT"
dt_name = prefix + str(ibits)
dtype = DataType[dt_name]
inp = helper.make_tensor_value_info("inp", TensorProto.FLOAT,
[n, c, idim, idim])
outp = helper.make_tensor_value_info("outp", TensorProto.FLOAT,
[n, c, odim, odim])
node = helper.make_node(
"QuantAvgPool2d",
["inp"],
["outp"],
domain="finn",
stride=s,
kernel=k,
ibits=ibits,
obits=obits,
signed=signed,
data_layout="NCHW",
)
graph = helper.make_graph(nodes=[node],
name="single-quantavgpool",
inputs=[inp],
outputs=[outp])
model = helper.make_model(graph)
model = ModelWrapper(model)
model = model.transform(InferShapes())
model = model.transform(InferDataTypes())
model = model.transform(InferDataLayouts())
model = model.transform(GiveUniqueNodeNames())
model = model.transform(GiveReadableTensorNames())
model_transformed = model.transform(ChangeDataLayoutQuantAvgPool2d())
model_transformed = model_transformed.transform(InferShapes())
model_transformed = model_transformed.transform(InferDataTypes())
model_transformed = model_transformed.transform(InferDataLayouts())
model_transformed = model_transformed.transform(GiveUniqueNodeNames())
model_transformed = model_transformed.transform(GiveReadableTensorNames())
inp_values = gen_finn_dt_tensor(dtype, [n, c, idim, idim])
idict = {"inp": inp_values}
assert oxe.compare_execution(model, model_transformed, idict)
assert len(model.graph.node) + 2 == len(model_transformed.graph.node)
assert model_transformed.graph.node[-1].op_type == "Transpose"
assert model_transformed.graph.node[0].op_type == "Transpose"
# check if QuantAvgPool2d node has datalayout set correctly
node = model_transformed.graph.node[1]
d_layout = get_by_name(node.attribute, "data_layout").s.decode("UTF-8")
assert d_layout == "NHWC"
assert model_transformed.get_tensor_layout(
node.input[0]) == DataLayout.NHWC
assert model_transformed.get_tensor_layout(
node.output[0]) == DataLayout.NHWC
def apply(self, model):
for node in model.graph.node:
op_type = node.op_type
if node.domain == "finn":
backend_attribute = util.get_by_name(node.attribute, "backend")
if backend_attribute is None:
continue
backend_value = backend_attribute.s.decode("UTF-8")
if backend_value == "fpgadataflow":
try:
# lookup op_type in registry of CustomOps
inst = registry.custom_op[op_type](node)
# ensure that code is generated
assert (
inst.get_nodeattr("code_gen_dir_ipgen") != ""
), """Node
attribute "code_gen_dir_ipgen" is empty. Please run
transformation CodeGen_ipgen first."""
# call the compilation function for this node
inst.ipgen_singlenode_code()
# ensure that executable path is now set
assert (
inst.get_nodeattr("ipgen_path") != ""
), """Transformation
HLSSynth_IPGen was not successful. Node attribute "ipgen_path"
is empty."""
except KeyError:
# exception if op_type is not supported
raise Exception(
"Custom op_type %s is currently not supported." % op_type
)
return (model, False)
def apply(self, model):
for node in model.graph.node:
op_type = node.op_type
if node.domain == "finn":
backend_attribute = util.get_by_name(node.attribute, "backend")
if backend_attribute is None:
continue
backend_value = backend_attribute.s.decode("UTF-8")
if backend_value == "fpgadataflow":
try:
# lookup op_type in registry of CustomOps
inst = registry.custom_op[op_type](node)
# find the IP gen dir
ipgen_path = inst.get_nodeattr("ipgen_path")
if ipgen_path is not None and os.path.isdir(
ipgen_path):
for dname, dirs, files in os.walk(ipgen_path):
for fname in files:
if fname.endswith(".v"):
fpath = os.path.join(dname, fname)
with open(fpath, "r") as f:
s = f.read()
old = '$readmemh(".'
new = '$readmemh("%s' % dname
s = s.replace(old, new)
with open(fpath, "w") as f:
f.write(s)
except KeyError:
pass
return (model, False)
def find_prod_mt(x):
is_mt = x.op_type == "MultiThreshold"
is_bp = False
if is_mt:
dt = get_by_name(x.attribute, "out_dtype").s
is_bp = dt.decode("utf-8") == "BIPOLAR"
return is_mt and is_bp
def inference_cost_upsample(model, node, discount_sparsity):
# extract info about the upsampling kernel attributes
mode = get_by_name(node.attribute, "mode").s.decode("utf-8")
scales_tensor = node.input[1]
scales_initializer = model.get_initializer(scales_tensor)
# extract info from tensor shapes and datatypes
(i_dtype, scale_dtype, o_dtype) = get_node_tensor_dtypes(model, node)
(i_shape, scale_shape, o_shape) = get_node_tensor_shapes(model, node)
bsize = i_shape[0]
ifm_ch = i_shape[1]
ofm_pix_total = np.prod(o_shape[2:])
# MAC calculation
if mode == "nearest":
# No calculation involved, since data is just copied over multiple times
n_macs = 0
elif mode == "linear":
# Data gets linearly interpolated in each dimension
# Two MACs per dimension and output pixel assumed
n_dim_scaling = np.sum(scales_initializer > 1)
n_macs = 2 * n_dim_scaling * ofm_pix_total * ifm_ch * bsize
else:
raise ValueError(f"Upsampling mode {mode} not supported for estimation.")
# Mem calculation
o_mem = np.prod(o_shape)
idt_name = i_dtype.name
odt_name = o_dtype.name
mac_op_type_str = "op_mac_%s_%s" % (idt_name, idt_name)
o_mem_type_str = "mem_o_%s" % (odt_name)
ret = {mac_op_type_str: n_macs, o_mem_type_str: o_mem}
return ret
def get_nodeattr(self, name):
"""Get a node attribute by name. Data is stored inside the ONNX node's
AttributeProto container. Attribute must be part of get_nodeattr_types.
Default value is returned if attribute is not set."""
try:
(dtype, req, def_val) = self.get_nodeattr_types()[name]
attr = get_by_name(self.onnx_node.attribute, name)
if attr is not None:
# dtype indicates which ONNX Attribute member to use
# (such as i, f, s...)
ret = attr.__getattribute__(dtype)
if dtype == "s":
# decode string attributes
ret = ret.decode("utf-8")
return ret
else:
if req:
raise Exception(
"""Required attribute %s unspecified in
a %s node"""
% (name, self.onnx_node.op_type)
)
else:
# not set, return default value
return def_val
except KeyError:
raise AttributeError("Op has no such attribute: " + name)
def apply(self, model):
graph = model.graph
node_ind = 0
graph_modified = False
for n in graph.node:
node_ind += 1
if (n.op_type in ["Add", "Sub"] and not model.is_fork_node(n)
and not model.is_join_node(n)):
A = model.get_initializer(n.input[1])
if (A is not None and np.isclose(
A, np.zeros_like(A), atol=self.atol).all()):
remove_node_and_rewire(model, n)
graph_modified = True
break
elif (n.op_type in ["Mul", "Div"] and not model.is_fork_node(n)
and not model.is_join_node(n)):
A = model.get_initializer(n.input[1])
if (A is not None and np.isclose(
A, np.ones_like(A), atol=self.atol).all()):
remove_node_and_rewire(model, n)
graph_modified = True
break
elif (n.op_type == "Pad" and not model.is_fork_node(n)
and not model.is_join_node(n)):
pads = get_by_name(n.attribute, "pads")
pads = np.asarray(pads.ints)
if (pads == 0).all():
remove_node_and_rewire(model, n)
graph_modified = True
break
model = model.transform(InferShapes())
return (model, graph_modified)
def apply(self, model):
graph = model.graph
node_ind = 0
graph_modified = False
nodes = [n for n in graph.node]
for n in nodes:
node_ind += 1
if n.op_type == "MaxPool" and not model.is_fork_node(n):
consumer = model.find_consumer(n.output[0])
pads = get_by_name(n.attribute, "pads")
has_padding = False
if pads is not None:
pads = list(pads.ints)
has_padding = np.prod(pads) != 0
if consumer is not None and consumer.op_type == "MultiThreshold":
mt_out = consumer.output[0]
mt_odt = model.get_tensor_datatype(mt_out)
if mt_odt.signed() and has_padding:
warnings.warn(
"Skipping padded MaxPool + signed-output MultiThreshold"
)
continue
# check for non-decreasing thresholds and nonnegative
# scale factor in MultiThreshold
# otherwise we cannot do the reordering
T = model.get_initializer(consumer.input[1])
T_sorted = np.sort(T, axis=1)
assert (T == T_sorted).all(
), "MultiThreshold must have non-decreasing thresholds"
mt_inst = getCustomOp(consumer)
if mt_inst.get_nodeattr("out_scale") < 0:
warnings.warn(
"Skipping MultiThreshold with negative out_scale")
continue
# remove old nodes
graph.node.remove(n)
graph.node.remove(consumer)
# swap conections
group_in = n.input[0]
# new tensor because dims change
group_middle = model.make_new_valueinfo_name()
group_out = consumer.output[0]
consumer.input[0] = group_in
consumer.output[0] = group_middle
n.input[0] = group_middle
n.output[0] = group_out
# insert them back in
graph.node.insert(node_ind - 1, consumer)
graph.node.insert(node_ind, n)
graph_modified = True
model = model.transform(InferShapes())
return (model, graph_modified)
def apply(self, model):
graph = model.graph
node_ind = 0
graph_modified = False
for n in graph.node:
node_ind += 1
if (
n.op_type == "Mul"
and not model.is_fork_node(n)
and not model.is_join_node(n)
):
consumer = model.find_consumer(n.output[0])
if (
consumer is not None
and consumer.op_type == "Conv"
and not model.is_join_node(consumer)
):
mul_weight_name = n.input[1]
A = model.get_initializer(mul_weight_name)
if A is None:
warnings.warn(
"""Mul weight tensor is not set. If it is a constant,
please use set_initializer to set the tensor."""
)
continue
conv_node = consumer
mul_node = n
start_name = mul_node.input[0]
conv_in_name = conv_node.input[0]
conv_in_shape = model.get_tensor_shape(conv_in_name)
ifm_ch = conv_in_shape[1]
group_attribute = get_by_name(consumer.attribute, "group")
if group_attribute is None:
continue
group_attribute = group_attribute.i
conv_out_name = conv_node.output[0]
conv_out_shape = model.get_tensor_shape(conv_out_name)
if A.shape == (1, ifm_ch, 1, 1) and ifm_ch == group_attribute:
# if the mul is channelwise and conv is depthwise,
# we can simply swap the order of ops
# rewire mul input to be conv input
conv_node.input[0] = start_name
model.set_tensor_shape(start_name, conv_in_shape)
model.set_tensor_datatype(start_name, DataType["FLOAT32"])
# use old conv input tensor as conv output
conv_node.output[0] = conv_in_name
model.set_tensor_shape(conv_in_name, conv_out_shape)
model.set_tensor_datatype(conv_in_name, DataType["FLOAT32"])
# use new conv output as new mul node input
mul_node.input[0] = conv_in_name
# use old conv output as new mul node output
mul_node.output[0] = conv_out_name
model.set_tensor_datatype(conv_out_name, DataType["FLOAT32"])
# move mul node past conv node
graph.node.remove(mul_node)
graph.node.insert(node_ind, mul_node)
graph_modified = True
model = model.transform(InferShapes())
return (model, graph_modified)
def get_metadata_prop(self, key):
"""Returns the value associated with metadata_prop with given key,
or None otherwise."""
metadata_prop = util.get_by_name(self.model.metadata_props, key, "key")
if metadata_prop is None:
return None
else:
return metadata_prop.value
def assign_partition_id(node):
if node.op_type in [
"GenericPartition", "StreamingDataflowPartition"
]:
return -1
else:
backend = get_by_name(node.attribute, "backend")
if backend is not None and backend.s.decode(
"UTF-8") == "fpgadataflow":
assigned_partition = get_by_name(node.attribute,
"partition_id")
if assigned_partition is not None:
return assigned_partition.i
else:
return 0
else:
return -1
def apply(self, model):
graph = model.graph
graph_modified = False
node_ind = 0
for n in graph.node:
node_ind += 1
if (
n.op_type == "Reshape"
and (model.get_initializer(n.input[1]) == [1, -1]).all()
) or n.op_type == "Flatten":
prod = model.find_producer(n.input[0])
if (
prod is not None
and prod.op_type == "Transpose"
# we ensure that the first dimension is not changed from the
# transpose operation
and get_by_name(prod.attribute, "perm").ints[0] == 0
):
data_layout = model.get_tensor_layout(prod.input[0])
# check for the data layout to interpret input shape correctly
if data_layout is None:
warnings.warn(
"""Data layout for input tensor of Transpose node is not set.
To use AbsorbTransposeIntoFlatten transformation
please set tensor data layout."""
)
continue
elif data_layout == DataLayout.NCHW:
(b, c, h, w) = model.get_tensor_shape(prod.input[0])
# if h=w=1 the transposition can be absorbed, otherwise
# the absorption would lead to an error in the behavior
if h != 1 or w != 1:
continue
# the flatten node from onnx keeps by default the first
# dim and flattens the rest, that is why this transformation
# can only work with b != 1 if the model contains already a
# flatten node and not a reshape node with shape = [1, -1].
# If the first dim of the input tensor is not 1, flatten and
# reshape (with shape = [1, -1]) would lead to different results
if n.op_type == "Reshape" and b != 1:
continue
elif data_layout == DataLayout.NHWC:
(b, h, w, c) = model.get_tensor_shape(prod.input[0])
if h != 1 or w != 1:
continue
if n.op_type == "Reshape" and b != 1:
continue
# create single flatten node and remove obsolete nodes
node = oh.make_node("Flatten", [prod.input[0]], [n.output[0]])
graph.node.remove(n)
graph.node.remove(prod)
graph.node.insert(node_ind, node)
graph_modified = True
if graph_modified:
model = model.transform(InferDataTypes())
return (model, graph_modified)
def set_metadata_prop(self, key, value):
"""Sets metadata property with given key to the given value."""
metadata_prop = util.get_by_name(self.model.metadata_props, key, "key")
if metadata_prop is None:
metadata_prop = onnx.StringStringEntryProto()
metadata_prop.key = key
metadata_prop.value = value
self.model.metadata_props.append(metadata_prop)
else:
metadata_prop.value = value
def apply(self, model):
for node in model.graph.node:
if node.domain == "finn":
backend_attribute = get_by_name(node.attribute, "backend")
if backend_attribute is None:
continue
backend_value = backend_attribute.s.decode("UTF-8")
if backend_value == "fpgadataflow":
_codegen_single_node(node, model, self.fpgapart, self.clk)
return (model, False)
def apply(self, model):
graph = model.graph
node_ind = 0
graph_modified = False
for n in graph.node:
node_ind += 1
if n.op_type == "Flatten":
consumer = model.find_consumer(n.output[0])
if consumer is not None and consumer.op_type == "TopK":
axis = get_by_name(consumer.attribute, "axis")
if axis is None or axis.i != -1:
continue
start_name = n.input[0]
data_layout = model.get_tensor_layout(start_name)
if data_layout != DataLayout.NHWC:
warnings.warn(
"""Transformation can't be applied. The input
to flatten has to have DataLayout.NHWC"""
)
continue
(b, h, w, c) = model.get_tensor_shape(start_name)
if h != 1 or w != 1:
continue
# get parameter k from topk
k = model.get_tensor_shape(consumer.output[1])[-1]
# swap conections
# new tensor because dims change
middle_name = model.make_new_valueinfo_name()
topk_indices = oh.make_tensor_value_info(
middle_name, TensorProto.INT64, [b, h, w, k]
)
end_name = consumer.output[1]
graph.value_info.append(topk_indices)
# remove old nodes
graph.node.remove(n)
graph.node.remove(consumer)
# set inputs and outputs correctly
consumer.input[0] = start_name
consumer.output[1] = middle_name
model.set_tensor_shape(consumer.output[0], (b, h, w, k))
n.input[0] = middle_name
n.output[0] = end_name
# insert them back in
graph.node.insert(node_ind - 1, consumer)
graph.node.insert(node_ind, n)
graph_modified = True
model = model.transform(InferShapes())
return (model, graph_modified)
def inference_cost_matmul(model, node, discount_sparsity):
# extract info from tensor shapes and datatypes
(i_dtype, w_dtype, o_dtype) = get_node_tensor_dtypes(model, node)
(i_shape, w_shape, o_shape) = get_node_tensor_shapes(model, node)
if node.op_type == "Gemm":
assert len(i_shape) == 2 and len(w_shape) == 2
tA = get_by_name(node.attribute, "transA")
tB = get_by_name(node.attribute, "transB")
if tA is not None and tA.i == 1:
i_shape = i_shape[::-1]
if tB is not None and tB.i == 1:
w_shape = w_shape[::-1]
# exclude common dim (last axis) from one side to avoid duplication
n_macs = np.prod(i_shape[:-1]) * np.prod(w_shape)
# deal with both dyn,param and dyn,dyn cases for weight memory
inp0_is_const = model.get_initializer(node.input[0]) is not None
inp1_is_const = model.get_initializer(node.input[1]) is not None
if inp0_is_const and (not inp1_is_const):
# inp 0 is static
w_mem = np.prod(i_shape)
wname = node.input[0]
elif (not inp0_is_const) and inp1_is_const:
# inp 1 is static
w_mem = np.prod(w_shape)
wname = node.input[1]
elif (not inp0_is_const) and (not inp1_is_const):
# both inputs dynamic
w_mem = 0
wname = None
if discount_sparsity and wname is not None:
density = get_node_weight_density(model, wname)
n_macs *= density
w_mem *= density
o_mem = np.prod(o_shape)
idt_name = i_dtype.name
wdt_name = w_dtype.name
odt_name = o_dtype.name
mac_op_type_str = "op_mac_%s_%s" % (idt_name, wdt_name)
w_mem_type_str = "mem_w_%s" % (wdt_name)
o_mem_type_str = "mem_o_%s" % (odt_name)
ret = {mac_op_type_str: n_macs, w_mem_type_str: w_mem, o_mem_type_str: o_mem}
return ret
def get_tensor_layout(self, tensor_name):
"""Returns the data layout annotation of tensor with given name.
The data layout is expressed as a list of strings with as many
elements as the number of dimensions in the tensor shape. Each
string annotates what is contained in that dimension. If there is no
data layout annotation, None will be returned.
Examples of data layout annotations:
["N", "C"] is tensor[batch][channel]
["N", "C", "H", "W"] is tensor[batch][channel][height][width]
["N", "H", "W", "C"] is tensor[batch][height][width][channel]
"""
graph = self._model_proto.graph
qnt_annotations = graph.quantization_annotation
ret = util.get_by_name(qnt_annotations, tensor_name, "tensor_name")
if ret is not None:
ret = util.get_by_name(ret.quant_parameter_tensor_names,
"tensor_layout", "key")
if ret is not None:
return eval(ret.value)
return None
def is_fpgadataflow_node(node):
"""Returns True if given node is fpgadataflow node. Otherwise False."""
is_node = False
if node is not None:
if node.domain == "finn":
n_backend = get_by_name(node.attribute, "backend")
if n_backend is not None:
backend_value = n_backend.s.decode("UTF-8")
if backend_value == "fpgadataflow":
is_node = True
return is_node
def test_code_gen_trafo():
idt = wdt = odt = DataType.BIPOLAR
mw = 8
mh = 8
pe = 4
simd = 4
inp = helper.make_tensor_value_info("inp", TensorProto.FLOAT, [1, mw])
outp = helper.make_tensor_value_info("outp", TensorProto.FLOAT, [1, mh])
node_inp_list = ["inp", "weights", "thresh"]
FCLayer_node = helper.make_node(
"StreamingFCLayer_Batch",
node_inp_list,
["outp"],
domain="finn",
backend="fpgadataflow",
code_gen_dir="",
executable_path="",
resType="ap_resource_lut()",
MW=mw,
MH=mh,
SIMD=simd,
PE=pe,
inputDataType=idt.name,
weightDataType=wdt.name,
outputDataType=odt.name,
noActivation=1,
)
graph = helper.make_graph(nodes=[FCLayer_node],
name="fclayer_graph",
inputs=[inp],
outputs=[outp])
model = helper.make_model(graph, producer_name="fclayer-model")
model = ModelWrapper(model)
model.set_tensor_datatype("inp", idt)
model.set_tensor_datatype("outp", odt)
model.set_tensor_datatype("weights", wdt)
W = util.gen_finn_dt_tensor(wdt, (mw, mh))
model.set_initializer("weights", W)
model = model.transform(CodeGen_npysim())
for node in model.graph.node:
code_gen_attribute = util.get_by_name(node.attribute,
"code_gen_dir_npysim")
tmp_dir = code_gen_attribute.s.decode("UTF-8")
assert os.path.isdir(
tmp_dir), """Code generation directory of node with
op type {} does not exist!""".format(node.op_type)
assert (len(os.listdir(tmp_dir)) !=
0), """Code generation directory of node with
op type {} is empty!""".format(node.op_type)
