def convert_unsupported_nodes_to_plugins(dynamic_graph): Input = gs.create_node("Input", op="Placeholder", dtype=tf.float32, shape=[1, 3, 300, 300]) PriorBox = gs.create_plugin_node(name="GridAnchor", op="GridAnchor_TRT", numLayers=6, minSize=0.2, maxSize=0.95, aspectRatios=[1.0, 2.0, 0.5, 3.0, 0.33], variance=[0.1, 0.1, 0.2, 0.2], featureMapShapes=[19, 10, 5, 3, 2, 1]) NMS = gs.create_plugin_node(name="NMS", op="NMS_TRT", shareLocation=1, varianceEncodedInTarget=0, backgroundLabelId=0, confidenceThreshold=1e-8, nmsThreshold=0.6, topK=100, keepTopK=100, numClasses=91, inputOrder=[0, 2, 1], confSigmoid=1, isNormalized=1) concat_priorbox = gs.create_node(name="concat_priorbox", op="ConcatV2", dtype=tf.float32, axis=2) concat_box_loc = gs.create_plugin_node("concat_box_loc", op="FlattenConcat_TRT", dtype=tf.float32, axis=1, ignoreBatch=0) concat_box_conf = gs.create_plugin_node("concat_box_conf", op="FlattenConcat_TRT", dtype=tf.float32, axis=1, ignoreBatch=0) namespace_plugin_map = { "MultipleGridAnchorGenerator": PriorBox, "Postprocessor": NMS, "Preprocessor": Input, "ToFloat": Input, "image_tensor": Input, "MultipleGridAnchorGenerator/Concatenate": concat_priorbox, "MultipleGridAnchorGenerator/Identity": concat_priorbox, "concat": concat_box_loc, "concat_1": concat_box_conf } # Now create a new graph by collapsing namespaces dynamic_graph.collapse_namespaces(namespace_plugin_map) # Remove the outputs, so we just have a single output node (NMS). dynamic_graph.remove(dynamic_graph.graph_outputs, remove_exclusive_dependencies=False) return dynamic_graph
def add_anchor_input(graph): """Add the missing const input for the GridAnchor node. Reference: 1. https://www.minds.ai/post/deploying-ssd-mobilenet-v2-on-the-nvidia-jetson-and-nano-platforms """ data = np.array([1, 1], dtype=np.float32) anchor_input = gs.create_node('AnchorInput', 'Const', value=data) graph.append(anchor_input) graph.find_nodes_by_op('GridAnchor_TRT')[0].input.insert(0, 'AnchorInput') return graph
def parse_gridAnchor(graph): """ define a constant input tensor and set that as the input for the GridAnchor node as UFF file does not provide an input element for the GridAnchor node """ data = np.array([1, 1], dtype=np.float32) anchor_input = gs.create_node("AnchorInput", "Const", value=data, dtype=tf.float32) graph.append(anchor_input) graph.find_nodes_by_op("GridAnchor_TRT")[0].input.insert(0, "AnchorInput") return graph
def preprocess(dynamic_graph): # Now create a new graph by collapsing namespaces dynamic_graph.collapse_namespaces(namespace_plugin_map) # Remove the outputs, so we just have a single output node (NMS). dynamic_graph.remove(dynamic_graph.graph_outputs, remove_exclusive_dependencies=False) # Create a constant Tensor and set it as input for GridAnchor_TRT data = np.array([1, 1], dtype=np.float32) anchor_input = gs.create_node("AnchorInput", "Const", value=data) graph.append(anchor_input) graph.find_nodes_by_op("GridAnchor_TRT")[0].input.insert(0, "AnchorInput")
def add_plugin(graph): all_assert_nodes = graph.find_nodes_by_op("Assert") graph.remove(all_assert_nodes, remove_exclusive_dependencies=True) all_identity_nodes = graph.find_nodes_by_op("Identity") graph.forward_inputs(all_identity_nodes) Input = gs.create_plugin_node( name="Input", op="Placeholder", shape=[1, 3, 300, 300] ) PriorBox = gs.create_plugin_node( name="GridAnchor", op="GridAnchor_TRT", minSize=0.2, maxSize=0.95, aspectRatios=[1.0, 2.0, 0.5, 3.0, 0.33], variance=[0.1,0.1,0.2,0.2], featureMapShapes=[19, 10, 5, 3, 2, 1], numLayers=6 ) NMS = gs.create_plugin_node( name="NMS", op="NMS_TRT", shareLocation=1, varianceEncodedInTarget=0, backgroundLabelId=0, confidenceThreshold=1e-8, nmsThreshold=0.6, topK=100, keepTopK=100, numClasses=91, inputOrder=[1, 0, 2], confSigmoid=1, isNormalized=1 ) concat_priorbox = gs.create_node( "concat_priorbox", op="ConcatV2", axis=2 ) concat_box_loc = gs.create_plugin_node( "concat_box_loc", op="FlattenConcat_TRT", ) concat_box_conf = gs.create_plugin_node( "concat_box_conf", op="FlattenConcat_TRT", ) namespace_plugin_map = { "MultipleGridAnchorGenerator": PriorBox, "Postprocessor": NMS, "Preprocessor": Input, "ToFloat": Input, "image_tensor": Input, "Concatenate": concat_priorbox, "concat": concat_box_loc, "concat_1": concat_box_conf } graph.collapse_namespaces(namespace_plugin_map) graph.remove(graph.graph_outputs, remove_exclusive_dependencies=False) graph.find_nodes_by_op("NMS_TRT")[0].input.remove("Input") return graph
def ssd_unsupported_nodes_to_plugin_nodes(ssd_graph): """Makes ssd_graph TensorRT comparible using graphsurgeon. This function takes ssd_graph, which contains graphsurgeon DynamicGraph data structure. This structure describes frozen Tensorflow graph, that can be modified using graphsurgeon (by deleting, adding, replacing certain nodes). The graph is modified by removing Tensorflow operations that are not supported by TensorRT's UffParser and replacing them with custom layer plugin nodes. Note: This specific implementation works only for ssd_inception_v2_coco_2017_11_17 network. Args: ssd_graph (gs.DynamicGraph): graph to convert Returns: gs.DynamicGraph: UffParser compatible SSD graph """ # Create TRT plugin nodes to replace unsupported ops in Tensorflow graph channels = ModelData.get_input_channels() height = ModelData.get_input_height() width = ModelData.get_input_width() Input = gs.create_plugin_node(name="Input", op="Placeholder", dtype=tf.float32, shape=[1, channels, height, width]) PriorBox = gs.create_plugin_node(name="GridAnchor", op="GridAnchor_TRT", minSize=0.2, maxSize=0.95, aspectRatios=[1.0, 2.0, 0.5, 3.0, 0.33], variance=[0.1, 0.1, 0.2, 0.2], featureMapShapes=[19, 10, 5, 3, 2, 1], numLayers=6) NMS = gs.create_plugin_node(name="NMS", op="NMS_TRT", shareLocation=1, varianceEncodedInTarget=0, backgroundLabelId=0, confidenceThreshold=1e-8, nmsThreshold=0.6, topK=100, keepTopK=100, numClasses=91, inputOrder=[0, 2, 1], confSigmoid=1, isNormalized=1) concat_priorbox = gs.create_node("concat_priorbox", op="ConcatV2", dtype=tf.float32, axis=2) concat_box_loc = gs.create_plugin_node("concat_box_loc", op="FlattenConcat_TRT", dtype=tf.float32, axis=1, ignoreBatch=0) concat_box_conf = gs.create_plugin_node("concat_box_conf", op="FlattenConcat_TRT", dtype=tf.float32, axis=1, ignoreBatch=0) # Create a mapping of namespace names -> plugin nodes. namespace_plugin_map = { "MultipleGridAnchorGenerator": PriorBox, "Postprocessor": NMS, "Preprocessor": Input, "ToFloat": Input, "image_tensor": Input, "MultipleGridAnchorGenerator/Concatenate": concat_priorbox, "MultipleGridAnchorGenerator/Identity": concat_priorbox, "concat": concat_box_loc, "concat_1": concat_box_conf } # Create a new graph by collapsing namespaces ssd_graph.collapse_namespaces(namespace_plugin_map) # Remove the outputs, so we just have a single output node (NMS). # If remove_exclusive_dependencies is True, the whole graph will be removed! ssd_graph.remove(ssd_graph.graph_outputs, remove_exclusive_dependencies=False) return ssd_graph
def add_plugin_and_preprocess(graph, model, num_classes, neuralet_adaptive_model): """add_plugin Reference: 1. https://github.com/AastaNV/TRT_object_detection/blob/master/config/model_ssd_mobilenet_v1_coco_2018_01_28.py 2. https://github.com/AastaNV/TRT_object_detection/blob/master/config/model_ssd_mobilenet_v2_coco_2018_03_29.py 3. https://devtalk.nvidia.com/default/topic/1050465/jetson-nano/how-to-write-config-py-for-converting-ssd-mobilenetv2-to-uff-format/post/5333033/#5333033 """ num_classes += 1 min_size = 0.2 max_size = 0.95 input_order = [0, 2, 1] if neuralet_adaptive_model == 0: input_order = [1, 0, 2] input_dims = (3, 300, 300) all_assert_nodes = graph.find_nodes_by_op("Assert") graph.remove(all_assert_nodes, remove_exclusive_dependencies=True) all_identity_nodes = graph.find_nodes_by_op("Identity") graph.forward_inputs(all_identity_nodes) Input = gs.create_plugin_node( name="Input", op="Placeholder", shape=(1,) + input_dims, dtype=tf.float32 ) PriorBox = gs.create_plugin_node( name="MultipleGridAnchorGenerator", op="GridAnchor_TRT", minSize=min_size, # was 0.2 maxSize=max_size, # was 0.95 aspectRatios=[1.0, 2.0, 0.5, 3.0, 0.33], variance=[0.1, 0.1, 0.2, 0.2], featureMapShapes=[19, 10, 5, 3, 2, 1], numLayers=6, dtype=tf.float32 ) NMS = gs.create_plugin_node( name="NMS", op="NMS_TRT", shareLocation=1, varianceEncodedInTarget=0, backgroundLabelId=0, confidenceThreshold=0.3, # was 1e-8 nmsThreshold=0.6, topK=100, keepTopK=100, numClasses=num_classes, # was 91 inputOrder=input_order, confSigmoid=1, isNormalized=1, dtype=tf.float32 ) concat_priorbox = gs.create_node( "concat_priorbox", op="ConcatV2", axis=2, dtype=tf.float32 ) if trt.__version__[0] >= '6': concat_box_loc = gs.create_plugin_node( "concat_box_loc", op="FlattenConcat_TRT", axis=1, ignoreBatch=0, dtype=tf.float32 ) concat_box_conf = gs.create_plugin_node( "concat_box_conf", op="FlattenConcat_TRT", axis=1, ignoreBatch=0, dtype=tf.float32 ) else: concat_box_loc = gs.create_plugin_node( "concat_box_loc", op="FlattenConcat_TRT", dtype=tf.float32 ) concat_box_conf = gs.create_plugin_node( "concat_box_conf", op="FlattenConcat_TRT", dtype=tf.float32 ) namespace_plugin_map = { "MultipleGridAnchorGenerator": PriorBox, "Postprocessor": NMS, "Preprocessor": Input, "Cast": Input, "ToFloat": Input, "image_tensor": Input, "MultipleGridAnchorGenerator/Concatenate": concat_priorbox, # for 'ssd_mobilenet_v1_coco' "Concatenate": concat_priorbox, "concat": concat_box_loc, "concat_1": concat_box_conf } graph.collapse_namespaces(namespace_plugin_map) graph.remove(graph.graph_outputs, remove_exclusive_dependencies=False) graph.find_nodes_by_op("NMS_TRT")[0].input.remove("Input") if ( "image_tensor:0" in graph.find_nodes_by_name("Input")[0].input ): # for ssd_mobilenet_v1 graph.find_nodes_by_name("Input")[0].input.remove("image_tensor:0") graph = replace_addv2(graph) graph = replace_fusedbnv3(graph) graph = parse_gridAnchor(graph) return graph
import graphsurgeon as gs import tensorflow as tf Input = gs.create_node("conv2d_input", op="Placeholder", dtype=tf.float32, shape=[-1, 1, 28, 28]) isrlu_activation = gs.create_node(name="activation_1", op="ISRLU", trt_plugin=True, dtype=tf.float32) namespace_plugin_map = {"activation_1": ISRLU} def preprocess(dynamic_graph): # Now create a new graph by collapsing namespaces dynamic_graph.collapse_namespaces(namespace_plugin_map) # Remove the outputs, so we just have a single output node dynamic_graph.remove(dynamic_graph.graph_outputs, remove_exclusive_dependencies=False)
def add_plugin(graph, model, spec): """add_plugin Reference: 1. https://github.com/AastaNV/TRT_object_detection/blob/master/config/model_ssd_mobilenet_v1_coco_2018_01_28.py 2. https://github.com/AastaNV/TRT_object_detection/blob/master/config/model_ssd_mobilenet_v2_coco_2018_03_29.py 3. https://devtalk.nvidia.com/default/topic/1050465/jetson-nano/how-to-write-config-py-for-converting-ssd-mobilenetv2-to-uff-format/post/5333033/#5333033 """ numClasses = spec['num_classes'] minSize = spec['min_size'] maxSize = spec['max_size'] inputOrder = spec['input_order'] all_assert_nodes = graph.find_nodes_by_op("Assert") graph.remove(all_assert_nodes, remove_exclusive_dependencies=True) all_identity_nodes = graph.find_nodes_by_op("Identity") graph.forward_inputs(all_identity_nodes) Input = gs.create_plugin_node( name="Input", op="Placeholder", shape=(1,) + INPUT_DIMS ) PriorBox = gs.create_plugin_node( name="MultipleGridAnchorGenerator", op="GridAnchor_TRT", minSize=minSize, # was 0.2 maxSize=maxSize, # was 0.95 aspectRatios=[1.0, 2.0, 0.5, 3.0, 0.33], variance=[0.1, 0.1, 0.2, 0.2], featureMapShapes=[19, 10, 5, 3, 2, 1], numLayers=6 ) NMS = gs.create_plugin_node( name="NMS", op="NMS_TRT", shareLocation=1, varianceEncodedInTarget=0, backgroundLabelId=0, confidenceThreshold=0.3, # was 1e-8 nmsThreshold=0.6, topK=100, keepTopK=100, numClasses=numClasses, # was 91 inputOrder=inputOrder, confSigmoid=1, isNormalized=1 ) concat_priorbox = gs.create_node( "concat_priorbox", op="ConcatV2", axis=2 ) concat_box_loc = gs.create_plugin_node( "concat_box_loc", op="FlattenConcat_TRT", ) concat_box_conf = gs.create_plugin_node( "concat_box_conf", op="FlattenConcat_TRT", ) namespace_plugin_map = { "MultipleGridAnchorGenerator": PriorBox, "Postprocessor": NMS, "Preprocessor": Input, "ToFloat": Input, "image_tensor": Input, "MultipleGridAnchorGenerator/Concatenate": concat_priorbox, # for 'ssd_mobilenet_v1_coco' "Concatenate": concat_priorbox, # for other models "concat": concat_box_loc, "concat_1": concat_box_conf } graph.collapse_namespaces(namespace_plugin_map) graph.remove(graph.graph_outputs, remove_exclusive_dependencies=False) graph.find_nodes_by_op("NMS_TRT")[0].input.remove("Input") if model == 'ssd_mobilenet_v1_coco': graph.find_nodes_by_name("Input")[0].input.remove("image_tensor:0") return graph
import graphsurgeon as gs import tensorflow as tf Input = gs.create_node("Input", op="Placeholder", dtype=tf.float32, shape=[1, 3, 512, 512]) PriorBox = gs.create_node("PriorBox", numLayers=6, minScale=0.2, maxScale=0.95, aspectRatios=[1.0, 2.0, 0.5, 3.0, 0.33], layerVariances=[0.1,0.1,0.2,0.2], #featureMapShapes=[19, 10, 5, 3, 2, 1]) featureMapShapes=[32, 16, 8, 4, 2, 1]) NMS = gs.create_node("NMS", scoreThreshold=1e-8, iouThreshold=0.6, maxDetectionsPerClass=100, maxTotalDetections=100, numClasses=91, scoreConverter="SIGMOID") concat_priorbox = gs.create_node("concat_priorbox", dtype = tf.float32, axis = 2) concat_box_loc = gs.create_node("concat_box_loc") concat_box_conf = gs.create_node("concat_box_conf") """ namespace_plugin_map = { "MultipleGridAnchorGenerator": PriorBox, "Postprocessor": NMS, "Preprocessor": Input, "ToFloat": Input,
def ssd_mobilenet_v1_unsupported_nodes_to_plugin_nodes(ssd_graph, input_shape): """Makes ssd_graph TensorRT comparible using graphsurgeon. This function takes ssd_graph, which contains graphsurgeon DynamicGraph data structure. This structure describes frozen Tensorflow graph, that can be modified using graphsurgeon (by deleting, adding, replacing certain nodes). The graph is modified by removing Tensorflow operations that are not supported by TensorRT's UffParser and replacing them with custom layer plugin nodes. Note: This specific implementation works only for ssd_mobilenet_v2_coco_2018_03_29 network. Args: ssd_graph (gs.DynamicGraph): graph to convert input_shape: input shape in CHW format Returns: gs.DynamicGraph: UffParser compatible SSD graph """ # Create TRT plugin nodes to replace unsupported ops in Tensorflow graph # channels = ModelData.get_input_channels() # height = ModelData.get_input_height() # width = ModelData.get_input_width() channels, height, width = input_shape Input = gs.create_plugin_node(name="Input", op="Placeholder", dtype=tf.float32, shape=[1, channels, height, width]) PriorBox = gs.create_plugin_node(name="GridAnchor", op="GridAnchor_TRT", minSize=0.2, maxSize=0.95, aspectRatios=[1.0, 2.0, 0.5, 3.0, 0.33], variance=[0.1, 0.1, 0.2, 0.2], featureMapShapes=[19, 10, 5, 3, 2, 1], numLayers=6 ) NMS = gs.create_plugin_node( name="NMS", op="NMS_TRT", shareLocation=1, varianceEncodedInTarget=0, backgroundLabelId=0, confidenceThreshold=1e-8, nmsThreshold=0.6, topK=100, keepTopK=100, numClasses=91, inputOrder=[0, 2, 1], confSigmoid=1, isNormalized=1 ) concat_priorbox = gs.create_node( "concat_priorbox", op="ConcatV2", dtype=tf.float32, axis=2 ) concat_box_loc = gs.create_plugin_node( "concat_box_loc", op="FlattenConcat_TRT", dtype=tf.float32, axis=1, ignoreBatch=0 ) concat_box_conf = gs.create_plugin_node( "concat_box_conf", op="FlattenConcat_TRT", dtype=tf.float32, axis=1, ignoreBatch=0 ) const = tf.NodeDef(name="Const", op="Const", attr={"dtype": tf.AttrValue(type=1), "value": tf.AttrValue(tensor=tf.make_tensor_proto([1, 1], dtype=tf.float32))}) death_list = ["strided_slice_7"] ssd_graph.remove(ssd_graph.find_nodes_by_path(death_list)) ssd_graph.find_nodes_by_path("Shape_6")[0].input.remove("Preprocessor/sub") # Create a mapping of namespace names -> plugin nodes. namespace_plugin_map = { "MultipleGridAnchorGenerator": concat_priorbox, "Postprocessor": NMS, "Preprocessor/map": Input, "ToFloat": Input, # "image_tensor": Input, "strided_slice_6": PriorBox, "Shape_6": const, "Concatenate": concat_priorbox, "concat": concat_box_loc, "concat_1": concat_box_conf } for node in ssd_graph.graph_inputs: namespace_plugin_map[node.name] = Input # Create a new graph by collapsing namespaces ssd_graph.collapse_namespaces(namespace_plugin_map) # Remove the outputs, so we just have a single output node (NMS). # If remove_exclusive_dependencies is True, the whole graph will be removed! ssd_graph.remove(ssd_graph.graph_outputs, remove_exclusive_dependencies=False) # Disconnect the Input node from NMS, as it expects to have only 3 inputs. to_remove_nodes = ["Input", "Preprocessor/stack_1"] for node in to_remove_nodes: if node in ssd_graph.find_nodes_by_op("NMS_TRT")[0].input: ssd_graph.find_nodes_by_op("NMS_TRT")[0].input.remove(node) if node == "Preprocessor/stack_1": ssd_graph.remove(node) ssd_graph.find_nodes_by_path("Input")[0].input.remove("image_tensor:0") return ssd_graph
def add_plugin(graph, input_dims, graph_chars=None): graph_def = graph.as_graph_def() if graph_chars is None: graph_chars = converter_util.GraphCharacteristics(graph_def) num_classes = converter_util.get_num_classes(graph_def, graph_chars=graph_chars) input_order = converter_util.get_NMS_input_order(graph_def, "Postprocessor", graph_chars=graph_chars) if any(x == -1 for x in input_order): print("NMS input order error: {} Aborting".format(input_order)) exit(1) if args.debug: print("Detected number of classes: ", num_classes) print("Detected NMS input order: ", input_order) assert_nodes = graph.find_nodes_by_op("Assert") graph.remove(assert_nodes, remove_exclusive_dependencies=True) identity_nodes = graph.find_nodes_by_op("Identity") graph.forward_inputs(identity_nodes) Input = gs.create_plugin_node(name="Input", op="Placeholder", shape=(1, ) + input_dims) # TODO: Consider automation of parameters PriorBox = gs.create_plugin_node(name="MultipleGridAnchorGenerator", op="GridAnchor_TRT", minSize=0.2, maxSize=0.95, aspectRatios=[1.0, 2.0, 0.5, 3.0, 0.33], variance=[0.1, 0.1, 0.2, 0.2], featureMapShapes=[19, 10, 5, 3, 2, 1], numLayers=6) NMS = gs.create_plugin_node(name="NMS", op="NMS_TRT", shareLocation=1, varianceEncodedInTarget=0, backgroundLabelId=0, confidenceThreshold=0.3, nmsThreshold=0.6, topK=100, keepTopK=100, numClasses=num_classes, inputOrder=input_order, confSigmoid=1, isNormalized=1) concat_box_loc = gs.create_plugin_node("concat_box_loc", op="FlattenConcat_TRT", axis=1, ignoreBatch=0) concat_box_conf = gs.create_plugin_node("concat_box_conf", op="FlattenConcat_TRT", axis=1, ignoreBatch=0) concat_priorbox = gs.create_node("concat_priorbox", op="ConcatV2", axis=2) namespace_map = { "MultipleGridAnchorGenerator": PriorBox, "Preprocessor": Input, "ToFloat": Input, "Cast": Input, "image_tensor": Input, "Postprocessor": NMS, "concat": concat_box_loc, "concat_1": concat_box_conf, "Concatenate": concat_priorbox, "MultipleGridAnchorGenerator/Concatenate": concat_priorbox, "SecondStagePostprocessor": NMS } graph.collapse_namespaces(namespace_map) graph.remove(graph.graph_outputs, remove_exclusive_dependencies=False) if graph.find_nodes_by_op("NMS_TRT"): if "Input" in graph.find_nodes_by_op("NMS_TRT")[0].input: graph.find_nodes_by_op("NMS_TRT")[0].input.remove("Input") if "image_tensor:0" in graph.find_nodes_by_name("Input")[0].input: graph.find_nodes_by_name("Input")[0].input.remove("image_tensor:0") if "image_tensor" in graph.find_nodes_by_name("Input")[0].input: graph.find_nodes_by_name("Input")[0].input.remove("image_tensor") if graph.find_nodes_by_name("ToFloat_3"): if "image_tensor:0" in graph.find_nodes_by_name("ToFloat_3")[0].input: graph.find_nodes_by_name("ToFloat_3")[0].input.remove( "image_tensor:0") return graph
def add_plugin(graph): all_assert_nodes = graph.find_nodes_by_op("Assert") graph.remove(all_assert_nodes, remove_exclusive_dependencies=True) all_identity_nodes = graph.find_nodes_by_op("Identity") graph.forward_inputs(all_identity_nodes) Input = gs.create_plugin_node(name="Input", op="Placeholder", dtype=tf.float32, shape=[None, 3, 300, 300]) PriorBox = gs.create_plugin_node( name="MultipleGridAnchorGenerator", op="GridAnchor_TRT", minSize=0.2, maxSize=0.95, aspectRatios=[1.0, 2.0, 0.5, 3.0, 0.33], variance=[0.1, 0.1, 0.2, 0.2], #featureMapShapes=[32, 16, 8, 4, 2, 1], nmsFeatureMapShapes=[19, 10, 5, 3, 2, 1], numLayers=6) NMS = gs.create_plugin_node( name="NMS", op="NMS_TRT", inputs=['concat_box_conf', 'Squeeze', 'concat_priorbox'], shareLocation=1, varianceEncodedInTarget=0, backgroundLabelId=0, confidenceThreshold=.015, nmsThreshold=0.4, topK=100, keepTopK=100, numClasses=39, # 38 object + 1 for unknown class inputOrder=[1, 0, 2], confSigmoid=1, isNormalized=1) concat_priorbox = gs.create_node("concat_priorbox", op="ConcatV2", dtype=tf.float32, inputs=['MultipleGridAnchorGenerator'], axis=2) concat_box_loc = gs.create_plugin_node("concat_box_loc", op="FlattenConcat_TRT", dtype=tf.float32, axis=1, ignoreBatch=0) concat_box_conf = gs.create_plugin_node("concat_box_conf", op="FlattenConcat_TRT", dtype=tf.float32, axis=1, ignoreBatch=0) # Create a dummy node in the 'MultipleGridAnchorGenerator' namespace. # This is a hack for 'ssd_mobilenet_v3_large/small'... if not any([ n.startswith('MultipleGridAnchorGenerator/') for n in graph.node_map.keys() ]): const = create_const_for_anchor_generator() dummy = gs.create_node( 'MultipleGridAnchorGenerator/dummy_for_anchors', op='Dummy', # not important here, node will be collapsed later inputs=['const_for_anchors']) graph.add(const) graph.add(dummy) namespace_plugin_map = { "MultipleGridAnchorGenerator": PriorBox, "Postprocessor": NMS, "Preprocessor": Input, "Cast": Input, "ToFloat": Input, # Maybe replaced by Cast? "image_tensor": Input, 'normalized_input_image_tensor': Input, 'MultipleGridAnchorGenerator/Concatenate': concat_priorbox, "Concatenate": concat_priorbox, "concat": concat_box_loc, "concat_1": concat_box_conf } graph.collapse_namespaces(namespace_plugin_map) if 'anchors' in [node.name for node in graph.graph_outputs]: graph.remove(graph.graph_outputs, remove_exclusive_dependencies=False) graph.find_nodes_by_op("NMS_TRT")[0].input.remove("Input") if 'NMS' not in [node.name for node in graph.graph_outputs]: graph.remove(graph.graph_outputs, remove_exclusive_dependencies=False) if 'NMS' not in [node.name for node in graph.graph_outputs]: # We expect 'NMS' to be one of the outputs raise RuntimeError('bad graph_outputs') if 'Input' in list(graph.find_nodes_by_name('NMS')[0].input): graph.find_nodes_by_name('NMS')[0].input.remove('Input') if 'image_tensor:0' in list(graph.find_nodes_by_name('Input')[0].input): graph.find_nodes_by_name('Input')[0].input.remove('image_tensor:0') graph = replace_addv2(graph) graph = replace_fusedbnv3(graph) return graph
import graphsurgeon as gs import tensorflow as tf Output = gs.create_node("dot_op_trt", op="my_matmul", dtype=tf.float32) namespace_plugin_map = {"dot_op": Output} def preprocess(dynamic_graph): # Now create a new graph by collapsing namespaces dynamic_graph.collapse_namespaces(namespace_plugin_map)
def add_plugin(cls, graph): import tensorflow as tf import graphsurgeon as gs all_assert_nodes = graph.find_nodes_by_op("Assert") graph.remove(all_assert_nodes, remove_exclusive_dependencies=True) all_identity_nodes = graph.find_nodes_by_op("Identity") graph.forward_inputs(all_identity_nodes) Input = gs.create_plugin_node(name="Input", op="Placeholder", dtype=tf.float32, shape=[1, *cls.INPUT_SHAPE]) PriorBox = gs.create_plugin_node( name="MultipleGridAnchorGenerator", op="GridAnchor_TRT", minSize=0.2, maxSize=0.95, aspectRatios=[1.0, 2.0, 0.5, 3.0, 0.33], variance=[0.1, 0.1, 0.2, 0.2], featureMapShapes=[19, 10, 5, 3, 2, 1], numLayers=6) NMS = gs.create_plugin_node(name="NMS", op="NMS_TRT", shareLocation=1, varianceEncodedInTarget=0, backgroundLabelId=0, confidenceThreshold=1e-8, nmsThreshold=cls.NMS_THRESH, topK=100, keepTopK=100, numClasses=91, inputOrder=[0, 2, 1], confSigmoid=1, isNormalized=1) concat_priorbox = gs.create_node("concat_priorbox", op="ConcatV2", dtype=tf.float32, axis=2) concat_box_loc = gs.create_plugin_node("concat_box_loc", op="FlattenConcat_TRT", dtype=tf.float32, axis=1, ignoreBatch=0) concat_box_conf = gs.create_plugin_node("concat_box_conf", op="FlattenConcat_TRT", dtype=tf.float32, axis=1, ignoreBatch=0) namespace_plugin_map = { "MultipleGridAnchorGenerator": PriorBox, "Postprocessor": NMS, "Preprocessor": Input, "ToFloat": Input, "image_tensor": Input, "MultipleGridAnchorGenerator/Concatenate": concat_priorbox, "concat": concat_box_loc, "concat_1": concat_box_conf } # Create a new graph by collapsing namespaces graph.collapse_namespaces(namespace_plugin_map) # Remove the outputs, so we just have a single output node (NMS). # If remove_exclusive_dependencies is True, the whole graph will be removed! graph.remove(graph.graph_outputs, remove_exclusive_dependencies=False) graph.find_nodes_by_op("NMS_TRT")[0].input.remove("Input") graph.find_nodes_by_name("Input")[0].input.remove("image_tensor:0") return graph
def add_plugin(graph): all_assert_nodes = graph.find_nodes_by_op("Assert") graph.remove(all_assert_nodes, remove_exclusive_dependencies=True) all_identity_nodes = graph.find_nodes_by_op("Identity") graph.forward_inputs(all_identity_nodes) Input = gs.create_plugin_node(name="Input", op="Placeholder", shape=[1, 3, 300, 300]) PriorBox = gs.create_plugin_node( name="GridAnchor", op="GridAnchor_TRT", minSize=0.2, maxSize=0.95, aspectRatios=[1.0, 2.0, 0.5, 3.0, 0.33], variance=[0.1, 0.1, 0.2, 0.2], featureMapShapes=[19, 10, 5, 3, 2, 1], # Resolution 300 #featureMapShapes=[29, 15, 8, 4, 2, 1], # Resolution 450 numLayers=6) NMS = gs.create_plugin_node(name="NMS", op="NMS_TRT", shareLocation=1, varianceEncodedInTarget=0, backgroundLabelId=0, confidenceThreshold=1e-8, nmsThreshold=0.6, topK=100, keepTopK=100, numClasses=2, inputOrder=[0, 2, 1], confSigmoid=1, isNormalized=1) concat_priorbox = gs.create_node("concat_priorbox", op="ConcatV2", axis=2) concat_box_loc = gs.create_plugin_node("concat_box_loc", op="FlattenConcat_TRT", axis=1, ignoreBatch=0) concat_box_conf = gs.create_plugin_node("concat_box_conf", op="FlattenConcat_TRT", axis=1, ignoreBatch=0) namespace_plugin_map = { "MultipleGridAnchorGenerator": PriorBox, "Postprocessor": NMS, "Preprocessor": Input, "Cast": Input, "image_tensor": Input, "Concatenate": concat_priorbox, "concat": concat_box_loc, "concat_1": concat_box_conf } for node in graph.find_nodes_by_op('FusedBatchNormV3'): gs.update_node(node, op='FusedBatchNorm') graph.collapse_namespaces(namespace_plugin_map) graph.remove(graph.graph_outputs, remove_exclusive_dependencies=False) graph.find_nodes_by_op("NMS_TRT")[0].input.remove("Input") # Create a constant Tensor and set it as input for GridAnchor_TRT data = np.array([1, 1], dtype=np.float32) anchor_input = gs.create_node("AnchorInput", "Const", value=data) graph.append(anchor_input) graph.find_nodes_by_op("GridAnchor_TRT")[0].input.insert(0, "AnchorInput") return graph
# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import graphsurgeon as gs import tensorflow as tf Input = gs.create_node("Input", op="Placeholder", dtype=tf.float32, shape=[1, 3, 300, 300]) PriorBox = gs.create_plugin_node(name="GridAnchor", op="GridAnchor_TRT", numLayers=6, minSize=0.2, maxSize=0.95, aspectRatios=[1.0, 2.0, 0.5, 3.0, 0.33], variance=[0.1, 0.1, 0.2, 0.2], featureMapShapes=[19, 10, 5, 3, 2, 1]) NMS = gs.create_plugin_node(name="NMS", op="NMS_TRT", shareLocation=1, varianceEncodedInTarget=0, backgroundLabelId=0, confidenceThreshold=1e-8,
NMS = gs.create_plugin_node(name="NMS", op="NMS_TRT", shareLocation=1, varianceEncodedInTarget=0, backgroundLabelId=0, confidenceThreshold=1e-8, nmsThreshold=0.6, topK=100, keepTopK=100, numClasses=91, inputOrder=[1, 0, 2], confSigmoid=1, isNormalized=1) concat_priorbox = gs.create_node(name="concat_priorbox", op="ConcatV2", dtype=tf.float32, axis=2) concat_box_loc = gs.create_plugin_node("concat_box_loc", op="FlattenConcat_TRT", dtype=tf.float32, axis=1, ignoreBatch=0) concat_box_conf = gs.create_plugin_node("concat_box_conf", op="FlattenConcat_TRT", dtype=tf.float32, axis=1, ignoreBatch=0) namespace_plugin_map = {
def ssd_pipeline_to_uff(checkpoint_path, config_path, tmp_dir='exported_model'): import graphsurgeon as gs from object_detection import exporter import tensorflow as tf import uff # TODO(@jwelsh): Implement by extending model builders with # TensorRT plugin stubs. Currently, this method uses pattern # matching which is a bit hacky and subject to fail when TF # object detection API exporter changes. We should add object # detection as submodule to avoid versioning incompatibilities. config = _load_config(config_path) frozen_graph_path = os.path.join(tmp_dir, FROZEN_GRAPH_NAME) # get input shape channels = 3 height = config.model.ssd.image_resizer.fixed_shape_resizer.height width = config.model.ssd.image_resizer.fixed_shape_resizer.width tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True # export checkpoint and config to frozen graph with tf.Session(config=tf_config) as tf_sess: with tf.Graph().as_default() as tf_graph: subprocess.call(['mkdir', '-p', tmp_dir]) exporter.export_inference_graph('image_tensor', config, checkpoint_path, tmp_dir, input_shape=[1, None, None, 3]) dynamic_graph = gs.DynamicGraph(frozen_graph_path) # remove all assert nodes #all_assert_nodes = dynamic_graph.find_nodes_by_op("Assert") #dynamic_graph.remove(all_assert_nodes, remove_exclusive_dependencies=True) # forward all identity nodes all_identity_nodes = dynamic_graph.find_nodes_by_op("Identity") dynamic_graph.forward_inputs(all_identity_nodes) # create input plugin input_plugin = gs.create_plugin_node(name=TRT_INPUT_NAME, op="Placeholder", dtype=tf.float32, shape=[1, height, width, channels]) # create anchor box generator anchor_generator_config = config.model.ssd.anchor_generator.ssd_anchor_generator box_coder_config = config.model.ssd.box_coder.faster_rcnn_box_coder priorbox_plugin = gs.create_plugin_node( name="priorbox", op="GridAnchor_TRT", minSize=anchor_generator_config.min_scale, maxSize=anchor_generator_config.max_scale, aspectRatios=list(anchor_generator_config.aspect_ratios), variance=[ 1.0 / box_coder_config.y_scale, 1.0 / box_coder_config.x_scale, 1.0 / box_coder_config.height_scale, 1.0 / box_coder_config.width_scale ], featureMapShapes=_get_feature_map_shape(config), numLayers=config.model.ssd.anchor_generator.ssd_anchor_generator. num_layers) # create nms plugin nms_config = config.model.ssd.post_processing.batch_non_max_suppression nms_plugin = gs.create_plugin_node( name=TRT_OUTPUT_NAME, op="NMS_TRT", shareLocation=1, varianceEncodedInTarget=0, backgroundLabelId=0, confidenceThreshold=nms_config.score_threshold, nmsThreshold=nms_config.iou_threshold, topK=nms_config.max_detections_per_class, keepTopK=nms_config.max_total_detections, numClasses=config.model.ssd.num_classes + 1, # add background inputOrder=[1, 2, 0], confSigmoid=1, isNormalized=1, scoreConverter="SIGMOID", codeType=3) priorbox_concat_plugin = gs.create_node("priorbox_concat", op="ConcatV2", dtype=tf.float32, axis=2) boxloc_concat_plugin = gs.create_plugin_node( "boxloc_concat", op="FlattenConcat_TRT_jetbot", dtype=tf.float32, ) boxconf_concat_plugin = gs.create_plugin_node( "boxconf_concat", op="FlattenConcat_TRT_jetbot", dtype=tf.float32, ) namespace_plugin_map = { "MultipleGridAnchorGenerator": priorbox_plugin, "Postprocessor": nms_plugin, "Preprocessor": input_plugin, "ToFloat": input_plugin, "image_tensor": input_plugin, "Concatenate": priorbox_concat_plugin, "concat": boxloc_concat_plugin, "concat_1": boxconf_concat_plugin } dynamic_graph.collapse_namespaces(namespace_plugin_map) # fix name for i, name in enumerate( dynamic_graph.find_nodes_by_op('NMS_TRT')[0].input): if TRT_INPUT_NAME in name: dynamic_graph.find_nodes_by_op('NMS_TRT')[0].input.pop(i) dynamic_graph.remove(dynamic_graph.graph_outputs, remove_exclusive_dependencies=False) uff_buffer = uff.from_tensorflow(dynamic_graph.as_graph_def(), [TRT_OUTPUT_NAME]) return uff_buffer
def add_plugin(graph, model, spec): """add_plugin Reference: 1. https://github.com/AastaNV/TRT_object_detection/blob/master/config/model_ssd_mobilenet_v1_coco_2018_01_28.py 2. https://github.com/AastaNV/TRT_object_detection/blob/master/config/model_ssd_mobilenet_v2_coco_2018_03_29.py 3. https://devtalk.nvidia.com/default/topic/1050465/jetson-nano/how-to-write-config-py-for-converting-ssd-mobilenetv2-to-uff-format/post/5333033/#5333033 """ numClasses = spec['num_classes'] minSize = spec['min_size'] maxSize = spec['max_size'] inputOrder = spec['input_order'] all_assert_nodes = graph.find_nodes_by_op('Assert') graph.remove(all_assert_nodes, remove_exclusive_dependencies=True) all_identity_nodes = graph.find_nodes_by_op('Identity') graph.forward_inputs(all_identity_nodes) Input = gs.create_plugin_node(name='Input', op='Placeholder', shape=(1, ) + INPUT_DIMS) PriorBox = gs.create_plugin_node( name='MultipleGridAnchorGenerator', op='GridAnchor_TRT', minSize=minSize, # was 0.2 maxSize=maxSize, # was 0.95 aspectRatios=[1.0, 2.0, 0.5, 3.0, 0.33], variance=[0.1, 0.1, 0.2, 0.2], featureMapShapes=[19, 10, 5, 3, 2, 1], numLayers=6) NMS = gs.create_plugin_node( name='NMS', op='NMS_TRT', shareLocation=1, varianceEncodedInTarget=0, backgroundLabelId=0, confidenceThreshold=0.3, # was 1e-8 nmsThreshold=0.6, topK=100, keepTopK=100, numClasses=numClasses, # was 91 inputOrder=inputOrder, confSigmoid=1, isNormalized=1) concat_priorbox = gs.create_node('concat_priorbox', op='ConcatV2', axis=2) if trt.__version__[0] >= '7': concat_box_loc = gs.create_plugin_node('concat_box_loc', op='FlattenConcat_TRT', axis=1, ignoreBatch=0) concat_box_conf = gs.create_plugin_node('concat_box_conf', op='FlattenConcat_TRT', axis=1, ignoreBatch=0) else: concat_box_loc = gs.create_plugin_node('concat_box_loc', op='FlattenConcat_TRT') concat_box_conf = gs.create_plugin_node('concat_box_conf', op='FlattenConcat_TRT') namespace_for_removal = [ 'ToFloat', 'image_tensor', 'Preprocessor/map/TensorArrayStack_1/TensorArrayGatherV3', ] namespace_plugin_map = { 'MultipleGridAnchorGenerator': PriorBox, 'Postprocessor': NMS, 'Preprocessor': Input, 'ToFloat': Input, 'Cast': Input, # added for models trained with tf 1.15+ 'image_tensor': Input, 'MultipleGridAnchorGenerator/Concatenate': concat_priorbox, # for 'ssd_mobilenet_v1_coco' 'Concatenate': concat_priorbox, # for other models 'concat': concat_box_loc, 'concat_1': concat_box_conf } graph.remove( graph.find_nodes_by_path( ['Preprocessor/map/TensorArrayStack_1/TensorArrayGatherV3']), remove_exclusive_dependencies=False) # for 'ssd_inception_v2_coco' graph.collapse_namespaces(namespace_plugin_map) graph = replace_addv2(graph) graph = replace_fusedbnv3(graph) if 'image_tensor:0' in graph.find_nodes_by_name('Input')[0].input: graph.find_nodes_by_name('Input')[0].input.remove('image_tensor:0') if 'Input' in graph.find_nodes_by_name('NMS')[0].input: graph.find_nodes_by_name('NMS')[0].input.remove('Input') # Remove the Squeeze to avoid "Assertion 'isPlugin(layerName)' failed" graph.forward_inputs( graph.find_node_inputs_by_name(graph.graph_outputs[0], 'Squeeze')) if 'anchors' in [node.name for node in graph.graph_outputs]: graph.remove('anchors', remove_exclusive_dependencies=False) if len(graph.find_nodes_by_op('GridAnchor_TRT')[0].input) < 1: graph = add_anchor_input(graph) if 'NMS' not in [node.name for node in graph.graph_outputs]: graph.remove(graph.graph_outputs, remove_exclusive_dependencies=False) if 'NMS' not in [node.name for node in graph.graph_outputs]: # We expect 'NMS' to be one of the outputs raise RuntimeError('bad graph_outputs') return graph
def add_plugin(graph: DynamicGraph, spec: ModelSpec): """add_plugin Reference: 1. https://github.com/AastaNV/TRT_object_detection/blob/master/config/model_ssd_mobilenet_v1_coco_2018_01_28.py 2. https://github.com/AastaNV/TRT_object_detection/blob/master/config/model_ssd_mobilenet_v2_coco_2018_03_29.py 3. https://devtalk.nvidia.com/default/topic/1050465/jetson-nano/how-to-write-config-py-for-converting-ssd-mobilenetv2-to-uff-format/post/5333033/#5333033 """ all_assert_nodes = graph.find_nodes_by_op("Assert") graph.remove(all_assert_nodes, remove_exclusive_dependencies=True) all_identity_nodes = graph.find_nodes_by_op("Identity") graph.forward_inputs(all_identity_nodes) input = create_plugin_node(name="Input", op="Placeholder", shape=(1, ) + spec.input_dim) prior_box = create_plugin_node( name="MultipleGridAnchorGenerator", op="GridAnchor_TRT", minSize=spec.min_size, # was 0.2 maxSize=spec.max_size, # was 0.95 aspectRatios=[1.0, 2.0, 0.5, 3.0, 0.33], variance=[0.1, 0.1, 0.2, 0.2], featureMapShapes=[19, 10, 5, 3, 2, 1], numLayers=6, ) nms = create_plugin_node( name="NMS", op="NMS_TRT", shareLocation=1, varianceEncodedInTarget=0, backgroundLabelId=0, confidenceThreshold=0.3, # was 1e-8 nmsThreshold=0.6, topK=100, keepTopK=100, numClasses=spec.num_classes, inputOrder=spec.input_order, confSigmoid=1, isNormalized=1, ) concat_priorbox = create_node("concat_priorbox", op="ConcatV2", axis=2) concat_box_loc = create_plugin_node("concat_box_loc", op="FlattenConcat_TRT", axis=1, ignoreBatch=0) concat_box_conf = create_plugin_node("concat_box_conf", op="FlattenConcat_TRT", axis=1, ignoreBatch=0) namespace_for_removal = [ "ToFloat", "image_tensor", "Preprocessor/map/TensorArrayStack_1/TensorArrayGatherV3", ] namespace_plugin_map = { "MultipleGridAnchorGenerator": prior_box, "Postprocessor": nms, "Preprocessor": input, "ToFloat": input, "Cast": input, # added for models trained with tf 1.15+ "image_tensor": input, "MultipleGridAnchorGenerator/Concatenate": concat_priorbox, # for 'ssd_mobilenet_v1_coco' "Concatenate": concat_priorbox, # for other models "concat": concat_box_loc, "concat_1": concat_box_conf, } graph.remove( graph.find_nodes_by_path( ["Preprocessor/map/TensorArrayStack_1/TensorArrayGatherV3"]), remove_exclusive_dependencies=False, ) # for 'ssd_inception_v2_coco' graph.collapse_namespaces(namespace_plugin_map) graph = replace_addv2(graph) graph = replace_fusedbnv3(graph) if "image_tensor:0" in graph.find_nodes_by_name("Input")[0].input: graph.find_nodes_by_name("Input")[0].input.remove("image_tensor:0") if "Input" in graph.find_nodes_by_name("NMS")[0].input: graph.find_nodes_by_name("NMS")[0].input.remove("Input") # Remove the Squeeze to avoid "Assertion 'isPlugin(layerName)' failed" graph.forward_inputs( graph.find_node_inputs_by_name(graph.graph_outputs[0], "Squeeze")) if "anchors" in [node.name for node in graph.graph_outputs]: graph.remove("anchors", remove_exclusive_dependencies=False) if len(graph.find_nodes_by_op("GridAnchor_TRT")[0].input) < 1: graph = add_anchor_input(graph) if "NMS" not in [node.name for node in graph.graph_outputs]: graph.remove(graph.graph_outputs, remove_exclusive_dependencies=False) if "NMS" not in [node.name for node in graph.graph_outputs]: # We expect 'NMS' to be one of the outputs raise RuntimeError("bad graph_outputs") return graph
def ssd_unsupported_nodes_to_plugin_nodes(ssd_graph, n_classes, input_dims, feature_dims): """Makes ssd_graph TensorRT comparible using graphsurgeon. This function takes ssd_graph, which contains graphsurgeon DynamicGraph data structure. This structure describes frozen Tensorflow graph, that can be modified using graphsurgeon (by deleting, adding, replacing certain nodes). The graph is modified by removing Tensorflow operations that are not supported by TensorRT's UffParser and replacing them with custom layer plugin nodes. Note: This specific implementation works only for ssd_inception_v2_coco_2017_11_17 network. Args: ssd_graph (gs.DynamicGraph): graph to convert Returns: gs.DynamicGraph: UffParser compatible SSD graph """ # Remove assert nodes all_assert_nodes = ssd_graph.find_nodes_by_op("Assert") # Remove those nodes from the graph. ssd_graph.remove(all_assert_nodes, remove_exclusive_dependencies=True) # Find all identity nodes. all_identity_nodes = ssd_graph.find_nodes_by_op("Identity") # Forward inputs those in the graph i.e. forward their inputs. ssd_graph.forward_inputs(all_identity_nodes) # Create TRT plugin nodes to replace unsupported ops in Tensorflow graph channels = input_dims[0] height = input_dims[1] width = input_dims[2] nodes = ssd_graph.node_map node_names = ssd_graph.node_map.keys() break_now = False class_predictor_label = 'concat_1' box_loc_label = 'concat' for inode in nodes['concat'].input: if break_now: break for jnode in nodes[inode].input: if break_now: break if 'ClassPredictor' in jnode: class_predictor_label = 'concat' box_loc_label = 'concat_1' break_now = True concat_namespace = "Concatenate" include_anchors = False for k in node_names: if "MultipleGridAnchorGenerator" in k: include_anchors = True if "MultipleGridAnchorGenerator/Concatenate" in k: concat_namespace = "MultipleGridAnchorGenerator/Concatenate" # Now we need to collapse a few namespaces. if include_anchors: Concat = gs.create_node("concat_priorbox", op="ConcatV2", dtype=tf.float32, axis=2) Input = gs.create_plugin_node(ModelData.INPUT_NAME, op="Placeholder", dtype=tf.float32, shape=[1, channels, height, width]) FlattenConcat_box_conf = gs.create_plugin_node( "concat_box_conf", op="FlattenConcat_TRT", dtype=tf.float32, ) FlattenConcat_box_loc = gs.create_plugin_node( "concat_box_loc", op="FlattenConcat_TRT", dtype=tf.float32, ) NMS = build_nms_node(numClasses=n_classes) # Create a mapping of namespace names -> plugin nodes. namespace_plugin_map = { "Postprocessor": NMS, "Preprocessor": Input, "ToFloat": Input, "image_tensor": Input, box_loc_label: FlattenConcat_box_loc, class_predictor_label: FlattenConcat_box_conf } # # Now create a new graph by collapsing namespaces # ssd_graph.collapse_namespaces(namespace_plugin_map) # # Determine the parameter for the GridAnchors # # print(ssd_graph.as_graph_def().node) # # print(tf.import_graph_def(ssd_graph.as_graph_def())) # for node in ssd_graph.as_graph_def().node: # if node.name == "BoxPredictor_2/BoxEncodingPredictor/BiasAdd": # print(type(node)) # print(node) # # print([dict(nodes[x].attr) for x in ssd_graph.node_map.keys() if 'BoxEncodingPredictor/BiasAdd' in x]) # # print(ssd_graph.find_nodes_by_name("BoxPredictor_2/BoxEncodingPredictor/BiasAdd")) # exit() if include_anchors: GridAnchor = build_grid_anchor_node(featureMapShapes=feature_dims) namespace_plugin_map[concat_namespace] = Concat namespace_plugin_map["MultipleGridAnchorGenerator"] = GridAnchor # Now create a new graph by collapsing namespaces ssd_graph.collapse_namespaces(namespace_plugin_map) # Remove the outputs, so we just have a single output node (NMS). # If remove_exclusive_dependencies is True, the whole graph will be removed! ssd_graph.remove(ssd_graph.graph_outputs, remove_exclusive_dependencies=False) # add in grid anchors for SSDLite if not include_anchors: Const = gs.create_node("Const", op="Const", dtype=tf.float32, value=[128, 128]) GridAnchor = build_grid_anchor_node(inputs=[Const], featureMapShapes=feature_dims) Concat = gs.create_node("concat_priorbox", inputs=[GridAnchor], op="ConcatV2", dtype=tf.float32, axis=2) ssd_graph.append(Const) ssd_graph.append(GridAnchor) ssd_graph.append(Concat) NMS = build_nms_node(inputs=list(NMS.input) + ["concat_priorbox"], numClasses=n_classes) namespace_plugin_map = {"NMS": NMS} ssd_graph.collapse_namespaces(namespace_plugin_map) # For exported graphs, we need to remove the squeeze node between concat plugin and the NMS plugin. # Downloaded graphs don't need this step. Refer to convert_ssd_v1.py all_squeeze_nodes = ssd_graph.find_nodes_by_name("Squeeze") # Forward inputs those in the graph i.e. forward their inputs. ssd_graph.forward_inputs(all_squeeze_nodes) # clean up NMS and Input nodes actualInputOrder = [] for node in ssd_graph._internal_graphdef.node: if node.name == "NMS": if ModelData.INPUT_NAME in node.input: node.input.remove(ModelData.INPUT_NAME) for input_name in node.input: if "loc" in input_name: actualInputOrder.append(0) elif "conf" in input_name: actualInputOrder.append(1) elif "priorbox" in input_name: actualInputOrder.append(2) elif node.name == ModelData.INPUT_NAME: if "image_tensor:0" in node.input: node.input.remove("image_tensor:0") # NOTE: since the actual order of the NMS nodes inputs differ between versions, I'll reinsert the NMS trt op NMS = build_nms_node(inputOrder=actualInputOrder, numClasses=n_classes) namespace_plugin_map = {"NMS": NMS} ssd_graph.collapse_namespaces(namespace_plugin_map) return ssd_graph