def test_activation():
in_q = QType(16, 13, True)
input_ = in_q.quantize(np.array([-1.2, 0.5, 0.5, -0.6])).reshape([4, 1, 1])
in_dims = Dim.named(c=4, h=1, w=1).impose_order(['c', 'h', 'w'])
params = ActivationParameters("test")
qrec = QuantizationRecord([in_q], [in_q])
out_dims = params.get_output_size([in_dims])
output_ = activation(params, in_dims, out_dims[0], input_, qrec=qrec)
output_ = in_q.dequantize(output_)
assert np.array_equal(output_, [[[0]], [[0.5]], [[0.5]], [[0]]])
def fuse_activation(G: NNGraph, tfl_opts, name: str, node: Node):
if tfl_opts.FusedActivationFunction(
) == ActivationFunctionType.ActivationFunctionType.NONE:
return add_node(G, node)
activation = TF_ACTIVATIONS[tfl_opts.FusedActivationFunction()]
anode = ActivationParameters(aname(name), activation)
return add_node(G, node, anode=anode)
def add_activation(G,
name,
subgraph,
op_name,
op,
load_tensors=False,
dequantize=False):
check(op.InputsLength() == 1,\
"Very odd " + str(op.InputsAsNumpy()))
activation = TF_ACTIVATION_OPERATORS[op_name]
return add_node(G, ActivationParameters(name, activation))
def fuse_activation(cls, tfl_opts, name, params, **kwargs):
G = kwargs['G']
opts = kwargs['opts']
ext = hashlib.sha1(name.encode(
"UTF-8")).hexdigest()[:8] if opts.get('anonymise') else 'activation'
if opts.get('load_quantization') and NodeId(params) in G.quantization:
node_qrec = G.quantization[NodeId(params)]
else:
node_qrec = None
# if node_qrec is not None and None in node_qrec.in_qs + node_qrec.out_qs:
# # one of the input is a constant or strange behaviour -> may be is something fusions will get rid of
# return add_node(self.G, node)
aparams = None
if tfl_opts.FusedActivationFunction() == ActivationFunctionType.NONE:
if node_qrec is not None and node_qrec.ktype.startswith('scaled'): # and opts.get('insert_relus'):
# here we have no activation in an asymmetric qtype -> may be an omitted relu
if node_qrec.out_qs[0] is not None and node_qrec.out_qs[0].min_val == 0:
if np.all(np.round(node_qrec.out_qs[0].max_val) == 6):
aparams = ActivationParameters.get_activation(
'relu6', name + f"_{ext}")
else:
aparams = ActivationParameters.get_activation(
'relu', name + f"_{ext}")
else:
aparams = ActivationParameters.get_activation(cls.TF_ACTIVATIONS[tfl_opts.FusedActivationFunction()],
name + f"_{ext}")
if aparams:
G.add_edge(NNEdge(from_node=params, to_node=aparams))
if opts.get('load_quantization'):
# In between the fused operation and activation the
# transfer is in int32 representation
node_qrec = G.quantization[NodeId(params)]
ina_qtype = deepcopy(node_qrec.out_qs[0])
outa_qtype = deepcopy(ina_qtype)
G.quantization[NodeId(aparams)] = QRec.scaled(
in_qs=[ina_qtype], out_qs=[outa_qtype])
params = aparams
return params
def add_shortcut(out_graph, routes, idx, l):
node_name = "{}_{}".format(l['type'], idx)
activation = l.get('activation')
if activation is None:
routes['in'][idx], routes['out'][idx] =\
out_graph.add_operator(node_name, MatrixAddParameters())
else:
activation = DARKNET_ACTIVATION_TYPES[activation]
routes['in'][idx], routes['out'][idx] =\
out_graph.add_operators(node_name,\
[MatrixAddParameters(), ActivationParameters(activation)])
routes['edges'].append([idx - 1, idx])
routes['edges'].append([int(l['from']) + idx, idx])
return False
def add_fully_connected(out_graph, routes, idx, l):
activation = get_str(l, 'activation', default="logistic")
filter_c = get_int(l, 'output')
node_name = "{}_{}".format(l['type'], idx)
if activation is None:
routes['in'][idx], routes['out'][idx] =\
out_graph.add_operator(node_name, FcParameters(FcFilterDim(filter_c), has_bias=True))
else:
activation = DARKNET_ACTIVATION_TYPES[activation]
routes['in'][idx], routes['out'][idx] =\
out_graph.add_operators(
node_name,
[FcParameters(FcFilterDim(filter_c), has_bias=True),\
ActivationParameters(activation)]
)
return True
def add_convolution(out_graph, routes, idx, l):
activation = get_str(l, 'activation', default="logistic")
node_name = "{}_{}".format(l['type'], idx)
routes['in'][idx] = node_name
padding = l.get("padding")
pad = l.get("pad")
size = get_int(l, 'size', 1)
groups = get_int(l, 'groups', 1)
filters_c = get_int(l, 'filters', 1)
stride = get_int(l, 'stride', 1)
batch_normalize = get_int(l, 'batch_normalize', 0)
flipped = get_int(l, 'flipped', 0)
custom = {'batch_normalize': batch_normalize == 1, 'flipped': flipped == 1}
assert 'binary' not in l, "Binary convolutions are not implemented"
assert 'xnor' not in l, "XNOR convolutions are not implemented"
assert 'dot' not in l, "dot is not implemented"
# padding calculation as per Darknet code
if pad is not None:
padding = int(size / 2)
if padding is None:
padding = 0
if activation is None:
routes['in'][idx], routes['out'][idx] =\
out_graph.add_operator(node_name,\
Conv2DParameters(Conv2DFilterDim(size, size, filters_c),\
StrideDim(stride), PadDim(padding), groups=groups,\
custom=custom, has_bias=True))
else:
activation = DARKNET_ACTIVATION_TYPES[activation]
routes['in'][idx], routes['out'][idx] =\
out_graph.add_operators(
node_name,
[
Conv2DParameters(Conv2DFilterDim(size, size, filters_c),\
StrideDim(stride), PadDim(padding), groups=groups,\
custom=custom, has_bias=True),
ActivationParameters(activation)
]
)
return True