def replace_sub_graph(self, graph: Graph, match: dict):
# node that is used to identify this pattern application instance for switching between supported
# and not supported LSTMCell sub-graphs; this value will be searched in __class__.instances_supported_by_IE.
anchor_node = match[__class__.anchor()]
assert anchor_node.has_valid('name'), \
'LSTMCell anchor node {} does\'t have attribute name; such nodes are not supported.'
match['input_op'] = match['concat'].in_node(0)
match['input_hidden_state'] = match['concat'].in_node(1)
match['input_cell_state'] = match['mul_0'].in_node(0) \
if match['mul_0'].in_node(0).id != match['sigmoid_0'].id else match['mul_0'].in_node(1)
pattern_edges = self.pattern()['edges']
pattern_edges.extend([('input_op', 'concat'),
('input_cell_state', 'mul_0'),
('input_hidden_state', 'concat')])
inputs = graph.get_inputs_with_ports(
match, pattern_edges, __class__.inputs + __class__.extra_inputs)
lstm_op = LSTMCell(
graph,
dict(
name=match['concat'].name + '/LSTMCell',
activations=None,
))
lstm_node = lstm_op.create_node(inputs)
lstm_node['old_infer'] = lstm_node.infer
lstm_node.infer = __class__.infer
# this node consumes one of the resulting LSTMCell outputs,
# it should be removed before reconnecting the nodes,
# otherwise it will be reconnected to the new cell output
graph.remove_node(match['tanh_1'].id)
for i, output in enumerate(__class__.outputs):
match[output].replace_node(lstm_node, i)
# Because of LSTMCell specification, this layer MUST have 2 outputs.
# => we need to create fake consumers for LSTMCell
# when this node haven't some outputs.
for i in [0, 1]:
if i not in lstm_node.out_nodes():
fake_output_node = Result(
graph, dict(name=lstm_node.name + "/Output_{}".format(i)))
fake_output_node.create_node(inputs=[lstm_node],
edge_attrs={
'out': i,
'in': 0
})
lstm_node['tf'] = True
lstm_node['extra_inputs'] = {
name: match[name].id
for name in __class__.extra_inputs
}
lstm_node['inputs'] = {
name: match[name].id
for name in __class__.inputs
}
def generate_sub_graph(self, graph: Graph, match: SubgraphMatch):
# IE DetectionOutput layer consumes flattened confidences and locations tensors.
# That is why we add reshapes before them.
locs_node = match.single_input_node(0)
conf_node = match.single_input_node(1)
prior_boxes_node = match.single_input_node(2)
locs_out_nodes = locs_node[0].out_nodes()
assert len(locs_out_nodes) == 1
locs_out_node = locs_out_nodes[list(locs_out_nodes.keys())[0]]
assert locs_out_node.op == "Result", locs_out_node.op
graph.remove_node(locs_out_node.id)
conf_out_nodes = conf_node[0].out_nodes()
assert len(conf_out_nodes) == 1
conf_out_node = conf_out_nodes[list(conf_out_nodes.keys())[0]]
assert conf_out_node.op == "Result", conf_out_node.op
graph.remove_node(conf_out_node.id)
# reshape operation to flatten confidence tensor
const = Const(graph, {'value': int64_array([0, -1])}).create_node()
reshape_loc_node = Reshape(graph, {}).create_node(
[locs_node, const], dict(name='DetectionOutput_Reshape_loc_'))
# reshape operation to flatten confidence tensor
reshape_conf_node = Reshape(graph, {}).create_node(
[conf_node, const], dict(name='DetectionOutput_Reshape_conf_'))
# remove the Result node after the priors node
assert prior_boxes_node[0].out_node().op == "Result"
graph.remove_node(prior_boxes_node[0].out_node().id)
# reshape operation for prior boxes tensor
const = Const(graph, {'value': int64_array([1, 2, -1])}).create_node()
reshape_priors_node = Reshape(graph, {}).create_node(
[prior_boxes_node, const],
dict(name='DetectionOutput_Reshape_priors_'))
# create Detection Output node with three inputs: locations, confidences and prior boxes
detection_output_op = DetectionOutput(
graph, match.custom_replacement_desc.custom_attributes)
detection_output_node = detection_output_op.create_node(
[reshape_loc_node, reshape_conf_node, reshape_priors_node],
dict(name=detection_output_op.attrs['type'] + '_'))
PermuteAttrs.set_permutation(reshape_priors_node,
detection_output_node, None)
# create Output node to mark DetectionOutput as a graph output operation
output_op = Result(graph)
output_op.create_node([detection_output_node], dict(name='sink_'))
return {}
def transform_graph(self, graph: Graph, replacement_descriptions: dict):
parameter_node = graph.get_op_nodes(op='Parameter')[0]
parameter_node['data_type'] = data_type_str_to_np(
parameter_node.graph.graph['cmd_params'].data_type)
# remove existing Result operations to remove unsupported sub-graph
graph.remove_nodes_from(
[node.id
for node in graph.get_op_nodes(op='Result')] + ['detections'])
# determine if the op which is a input/final result of mean value and scale applying to the input tensor
# then connect it to the input of the first convolution of the model, so we remove the image pre-processing
# which includes padding and resizing from the model
preprocessing_input_node_id = replacement_descriptions[
'preprocessing_input_node']
assert preprocessing_input_node_id in graph.nodes, 'The node with name "{}" is not found in the graph. This ' \
'should be a last node before image normalization and is specified' \
' in the json file.'.format(preprocessing_input_node_id)
preprocessing_input_node = Node(graph, preprocessing_input_node_id)
consumer_node = preprocessing_input_node.out_port(
0).get_connection().get_destination().node
consumer_node.in_port(0).get_connection().set_source(
parameter_node.out_port(0))
preprocessing_output_node_id = replacement_descriptions[
'preprocessing_output_node']
assert preprocessing_output_node_id in graph.nodes, 'The node with name "{}" is not found in the graph. This ' \
'node should provide scaled image output and is specified' \
' in the json file.'.format(preprocessing_output_node_id)
preprocessing_output_node = Node(graph, preprocessing_output_node_id)
preprocessing_output_node.out_port(0).disconnect()
convolution_nodes = [
n for n in graph.pseudo_topological_sort()
if n.soft_get('type') == 'Convolution'
]
convolution_nodes[0].in_port(0).get_connection().set_source(
preprocessing_output_node.out_port(0))
# create prior boxes (anchors) generator
aspect_ratios = replacement_descriptions['aspect_ratios']
assert len(aspect_ratios) % 2 == 0
aspect_ratios = list(zip(aspect_ratios[::2], aspect_ratios[1::2]))
priors_generator = self.AnchorGenerator(
min_level=int(replacement_descriptions['min_level']),
aspect_ratios=aspect_ratios,
num_scales=int(replacement_descriptions['num_scales']),
anchor_scale=replacement_descriptions['anchor_scale'])
prior_boxes = []
for i in range(100):
inp_name = 'box_net/box-predict{}/BiasAdd'.format('_%d' %
i if i else '')
if inp_name not in graph:
break
widths, heights = priors_generator.get(i)
prior_box_op = PriorBoxClusteredOp(
graph, {
'width': mo_array(widths),
'height': mo_array(heights),
'clip': 0,
'flip': 0,
'variance': replacement_descriptions['variance'],
'offset': 0.5
})
prior_boxes.append(
prior_box_op.create_node(
[Node(graph, inp_name), parameter_node]))
# concatenate prior box operations
concat_prior_boxes = Concat(graph, {'axis': -1}).create_node()
for idx, node in enumerate(prior_boxes):
concat_prior_boxes.add_input_port(idx)
concat_prior_boxes.in_port(idx).connect(node.out_port(0))
conf = Sigmoid(graph, dict(name='concat/sigmoid')).create_node(
[Node(graph, 'concat')])
reshape_size_node = Const(graph, {
'value': int64_array([0, -1])
}).create_node([])
logits = Reshape(graph, dict(name=conf.name + '/Flatten')).create_node(
[conf, reshape_size_node])
deltas = Reshape(graph, dict(name='concat_1/Flatten')).create_node(
[Node(graph, 'concat_1'), reshape_size_node])
# revert convolution boxes prediction weights from yxYX to xyXY (convolutions share weights and bias)
weights = Node(graph, 'box_net/box-predict/pointwise_kernel')
weights.value = weights.value.reshape(-1, 4)[:, [1, 0, 3, 2]].reshape(
weights.shape)
bias = Node(graph, 'box_net/box-predict/bias')
bias.value = bias.value.reshape(-1,
4)[:, [1, 0, 3, 2]].reshape(bias.shape)
detection_output_node = DetectionOutput(
graph,
dict(
name='detections',
share_location=1,
background_label_id=int(
replacement_descriptions['num_classes']) + 1,
nms_threshold=replacement_descriptions['nms_threshold'],
confidence_threshold=replacement_descriptions[
'confidence_threshold'],
top_k=100,
keep_top_k=100,
code_type='caffe.PriorBoxParameter.CENTER_SIZE',
)).create_node([deltas, logits, concat_prior_boxes])
output_op = Result(graph, dict(name='output'))
output_op.create_node([detection_output_node])
