def GetParams(self):
"""Create a graph containing multiple segment."""
input_name = "input"
input_dims = [2, 32, 32, 3]
output_name = "output"
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(
dtype=dtypes.float32, shape=input_dims, name=input_name)
with g.device("/GPU:0"):
n = inp
c = constant_op.constant(1.0, name="c")
# Adds control dependency from the constant op to a trt incompatible op,
# and adds control dependency from the trt incompatible op to all other
# ops, to make sure the constant op cannot be contracted with any trt
# segment that depends on it.
with g.control_dependencies([c]):
d = self.trt_incompatible_op(n, name="incompatible")
with g.control_dependencies([d]):
n = math_ops.add(n, c, name="add")
n = math_ops.mul(n, n, name="mul")
n = math_ops.add(n, n, name="add1")
n = self.trt_incompatible_op(n, name="incompatible1")
with g.control_dependencies([d]):
n = math_ops.add(n, c, name="add2")
n = math_ops.mul(n, n, name="mul1")
n = math_ops.add(n, n, name="add3")
array_ops.squeeze(n, name=output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[tuple(input_dims)])
def GetParams(self):
"""Create a graph containing multiple segment."""
input_name = "input"
input_dims = [2, 32, 32, 3]
output_name = "output"
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(
dtype=dtypes.float32, shape=input_dims, name=input_name)
with g.device("/GPU:0"):
n = inp
c = constant_op.constant(1.0, name="c")
n = math_ops.add(n, c, name="add")
n = math_ops.mul(n, n, name="mul")
n = math_ops.add(n, n, name="add1")
n = self.trt_incompatible_op(n, name="incompatible1")
n = math_ops.add(n, c, name="add2")
n = math_ops.mul(n, n, name="mul1")
n = math_ops.add(n, n, name="add3")
array_ops.squeeze(n, name=output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[tuple(input_dims)])
def GetParams(self):
"""Single vgg layer test in TF-TRT conversion."""
dtype = dtypes.float32
input_name = "input"
input_dims = [5, 8, 8, 2]
output_name = "output"
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
x, _, _ = nn_impl.fused_batch_norm(
x, [1.0, 1.0], [0.0, 0.0],
mean=[0.5, 0.5],
variance=[1.0, 1.0],
is_training=False)
e = constant_op.constant(
np.random.randn(1, 1, 2, 6), name="weights", dtype=dtype)
conv = nn.conv2d(
input=x, filter=e, strides=[1, 2, 2, 1], padding="SAME", name="conv")
b = constant_op.constant(np.random.randn(6), name="bias", dtype=dtype)
t = nn.bias_add(conv, b, name="biasAdd")
relu = nn.relu(t, "relu")
idty = array_ops.identity(relu, "ID")
v = nn_ops.max_pool(
idty, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool")
array_ops.squeeze(v, name=output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[(5, 2, 2, 6)])
def GetParams(self):
"""Create a graph containing two segment."""
input_name = "input"
input_dims = [2, 32, 32, 3]
output_name = "output"
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(
dtype=dtypes.float32, shape=input_dims, name=input_name)
with g.device("/GPU:0"):
n = inp
for i in range(2):
c = constant_op.constant(1.0, name="c%d" % i)
n = math_ops.add(n, c, name="add%d" % i)
n = math_ops.mul(n, n, name="mul%d" % i)
edge = self.trt_incompatible_op(n, name="incompatible")
with g.control_dependencies([edge]):
c = constant_op.constant(1.0, name="c2")
n = math_ops.add(n, c, name="add2")
n = math_ops.mul(n, n, name="mul2")
c = constant_op.constant(1.0, name="c3")
n = math_ops.add(n, c, name="add3")
n = math_ops.mul(n, n, name="mul3")
array_ops.squeeze(n, name=output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[tuple(input_dims)])
def GetParams(self):
"""Test for Constant broadcasting in TF-TRT."""
dtype = dtypes.float32
input_name = 'input'
input_dims = [5, 12, 12, 2]
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
filt1 = constant_op.constant(
0.3, shape=(3, 3, 2, 1), dtype=dtype, name='filt1')
y1 = nn.conv2d(x, filt1, strides=[1, 1, 1, 1], padding='SAME', name='y1')
z1 = nn.relu(y1, name='z1')
filt2 = constant_op.constant(
np.random.randn(9), shape=(3, 3, 1, 1), dtype=dtype, name='filt2')
y2 = nn.conv2d(z1, filt2, strides=[1, 1, 1, 1], padding='SAME', name='y2')
z2 = nn.relu(y2, name='z')
filt3 = constant_op.constant(
np.random.randn(3, 3, 1, 1),
shape=(3, 3, 1, 1),
dtype=dtype,
name='filt3')
y3 = nn.conv2d(z2, filt3, strides=[1, 1, 1, 1], padding='SAME', name='y3')
nn.relu(y3, name='output')
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
num_expected_engines=1,
expected_output_dims=(5, 12, 12, 1),
allclose_atol=1.e-02,
allclose_rtol=1.e-02)
def GetParams(self):
"""Neighboring node wiring tests in TF-TRT conversion."""
dtype = dtypes.float32
input_name = "input"
input_dims = [2, 3, 7, 5]
output_name = "output"
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype,
shape=input_dims,
name=input_name)
e = constant_op.constant(np.random.normal(.3, 0.05, [3, 2, 3, 4]),
name="weights",
dtype=dtype)
conv = nn.conv2d(input=x,
filter=e,
data_format="NCHW",
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
b = constant_op.constant(np.random.normal(1.0, 1.0, [1, 4, 1, 1]),
name="bias",
dtype=dtype)
t = math_ops.mul(conv, b, name="mul")
e = self.trt_incompatible_op(conv, name="incompatible")
t = math_ops.sub(t, e, name="sub")
array_ops.squeeze(t, name=output_name)
return trt_test.TfTrtIntegrationTestParams(gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[
(2, 4, 5, 4)
])
def GetParams(self):
"""Create a graph containing two segment."""
input_name = "input"
input_dims = [2, 32, 32, 3]
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(dtype=dtypes.float32,
shape=input_dims,
name=input_name)
with g.device("/GPU:0"):
n = inp
for i in range(2):
c = constant_op.constant(1.0, name="c%d" % i)
n = math_ops.add(n, c, name="add%d" % i)
n = math_ops.mul(n, n, name="mul%d" % i)
edge = self.trt_incompatible_op(n, name="incompatible")
with g.control_dependencies([edge]):
c = constant_op.constant(1.0, name="c2")
n = math_ops.add(n, c, name="add2")
n = math_ops.mul(n, n, name="mul2")
c = constant_op.constant(1.0, name="c3")
n = math_ops.add(n, c, name="add3")
n = math_ops.mul(n, n, name="mul3")
array_ops.squeeze(n, name=self.output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
expected_engines={
# Only the first engine is built.
"my_trt_op_0": ["c0", "c1", "add0", "add1", "mul0", "mul1"]
},
expected_output_dims=tuple(input_dims),
allclose_atol=1.e-06,
allclose_rtol=1.e-06)
def GetParams(self):
"""Testing engine with the same tensor repeated as output via identity."""
input_name = 'input'
input_dims = [100, 32]
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtypes.float32,
shape=input_dims,
name=input_name)
b = self._ConstOp((32, 4))
x1 = math_ops.matmul(x, b)
b = self._ConstOp((1, 4))
x1 = x1 + b
out1 = array_ops.identity(x1, name='output1')
out2 = array_ops.identity(x1, name='output2')
iden1 = array_ops.identity(x1)
out3 = array_ops.identity(iden1, name='output3')
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=['output1', 'output2', 'output3'],
expected_output_dims=[(100, 4), (100, 4), (100, 4)])
def GetParams(self):
"""Testing that output type of engine using Top-K is set correctly."""
dtype = dtypes.float32
input_name = "input"
input_dims = [100, 100]
k = 5
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype,
shape=input_dims,
name=input_name)
k_tensor = constant_op.constant(k,
dtype=dtypes.int32,
name="Const")
values, indices = nn_ops.top_k(x, k_tensor, name="TopK")
# Reshape will act as a layer between the TopK output and the engine
# output, requiring the output tensor of reshape to be set explicitly to
# int32.
indices = array_ops.reshape(indices, [100, 1, 5], name="Reshape")
values = array_ops.identity(values, name="output_values")
indices = array_ops.identity(indices, name="output_indices")
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[[input_dims]],
output_names=["output_values", "output_indices"],
expected_output_dims=[[[100, k], [100, 1, k]]])
def GetParams(self):
"""Testing conversion of BatchMatMul in TF-TRT conversion."""
dtype = dtypes.float32
input_name = "input"
input_dims = [2, 15, 15, 3]
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(
dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
with g.device("/GPU:0"):
e1 = constant_op.constant(
np.random.randn(1, 1, 3, 5), name="kernel_1", dtype=dtype)
e2 = constant_op.constant(
np.random.randn(1, 1, 5, 10), name="kernel_2", dtype=dtype)
conv = nn.conv2d(
input=inp,
filter=e1,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
out = nn.conv2d(
input=conv,
filter=e2,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv_2")
array_ops.squeeze(out, name=self.output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
num_expected_engines=1,
expected_output_dims=(2, 15, 15, 10),
allclose_atol=1.e-02,
allclose_rtol=1.e-02)
def GetParams(self):
dtype = dtypes.float32
input_name = "input"
input_dims = [[[1, 10, 10, 2]], [[2, 10, 10, 2]], [[4, 10, 10, 2]],
[[2, 10, 10, 2]]]
expected_output_dims = [[[1, 10, 10, 1]], [[2, 10, 10, 1]], [[4, 10, 10,
1]],
[[2, 10, 10, 1]]]
output_name = "output"
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(
dtype=dtype, shape=[None, 10, 10, 2], name=input_name)
conv_filter = constant_op.constant(
np.random.randn(3, 3, 2, 1), dtype=dtypes.float32)
x = nn.conv2d(
input=x,
filter=conv_filter,
strides=[1, 1, 1, 1],
padding="SAME",
name="conv")
bias = constant_op.constant(
np.random.randn(1, 10, 10, 1), dtype=dtypes.float32)
x = math_ops.add(x, bias)
x = nn.relu(x)
x = array_ops.identity(x, name="output")
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=input_dims,
output_names=[output_name],
expected_output_dims=expected_output_dims)
def GetParams(self):
"""Create a graph containing multiple segment."""
input_name = "input"
input_dims = [2, 32, 32, 3]
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(dtype=dtypes.float32,
shape=input_dims,
name=input_name)
with g.device("/GPU:0"):
n = inp
c = constant_op.constant(1.0, name="c")
n = math_ops.add(n, c, name="add")
n = math_ops.mul(n, n, name="mul")
n = math_ops.add(n, n, name="add1")
n = self.trt_incompatible_op(n, name="incompatible1")
n = math_ops.add(n, c, name="add2")
n = math_ops.mul(n, n, name="mul1")
n = math_ops.add(n, n, name="add3")
array_ops.squeeze(n, name=self.output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
expected_engines={
"my_trt_op_0": ["add2", "add3", "mul1"],
# Why segment ["add", "add1", "mul"] was assigned segment id 1
# instead of 0: the parent node of this segment is actually const
# node 'c', but it's removed later since it's const output of the
# segment which is not allowed.
"my_trt_op_1": ["add", "add1", "mul"]
},
expected_output_dims=tuple(input_dims),
allclose_atol=1.e-06,
allclose_rtol=1.e-06)
def GetParams(self):
"""Neighboring node wiring tests in TF-TRT conversion."""
dtype = dtypes.float32
input_name = "input"
input_dims = [2, 3, 7, 5]
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype,
shape=input_dims,
name=input_name)
e = constant_op.constant(np.random.normal(.3, 0.05, [3, 2, 3, 4]),
name="weights",
dtype=dtype)
conv = nn.conv2d(input=x,
filter=e,
data_format="NCHW",
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
b = constant_op.constant(np.random.normal(1.0, 1.0, [1, 4, 1, 1]),
name="bias",
dtype=dtype)
t = conv * b
e = gen_math_ops.tan(conv)
t = t - e
array_ops.squeeze(t, name=self.output_name)
return trt_test.TfTrtIntegrationTestParams(gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
num_expected_engines=2,
expected_output_dims=(2, 4,
5, 4),
allclose_atol=1.e-03,
allclose_rtol=1.e-03)
def GetParams(self):
"""Testing conversion of conv2d_transpose (AKA Conv2DBackpropInput)"""
np.random.seed(1234)
dtype = dtypes.float32
input_name = "input"
n, c, h, w = 13, 3, 7, 11
num_filters = 8
input_dims = [n, c, h, w]
output_name = "output"
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(
dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
with g.device("/GPU:0"):
weights_shape = [2, 2, num_filters, c]
weights = constant_op.constant(
np.random.randn(*weights_shape), dtype=dtype)
output_shape = constant_op.constant([n, num_filters, h * 2, w * 2],
dtype=dtypes.int32)
output = nn_ops.conv2d_transpose(
inp,
weights,
output_shape,
strides=[1, 1, 2, 2],
padding="SAME",
data_format="NCHW")
output = array_ops.identity(output, name=output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[[input_dims]],
output_names=[output_name],
expected_output_dims=[[[n, num_filters, h * 2, w * 2]]])
def GetParams(self):
"""Tests for scale & elementwise layers in TF-TRT."""
input_name = "input"
input_dims = [10, 24, 24, 20]
output_name = "output"
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtypes.float32,
shape=input_dims,
name=input_name)
for weights_shape in [
(1, ), # scale
(24, 1, 1), # scale
(24, 24, 20), # scale
(20, ), # elementwise
(1, 24, 1, 1), # elementwise
(1, 24, 24, 1), # elementwise
(1, 24, 24, 20), # elementwise
(24, 20), # elementwise
]:
a = self._ConstOp(weights_shape)
f = x + a
x = math_ops.sigmoid(f)
a = self._ConstOp(weights_shape)
f = a + x
x = math_ops.sigmoid(f)
gen_array_ops.reshape(x, [5, -1], name=output_name)
return trt_test.TfTrtIntegrationTestParams(gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[
(5, 23040)
])
def _GetParams(add_quantization_nodes, dtype=dtypes.float32):
input_name = "input"
input_dims = [8, 8]
output_name = "output"
def _Quantize(x, r):
if add_quantization_nodes:
x = gen_array_ops.fake_quant_with_min_max_vars(x, -r, r)
return x
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype,
shape=[None] + input_dims[1:],
name=input_name)
x = _Quantize(x, 10.0)
x = x + 5
x = _Quantize(x, 15.0)
x = x - 5
x = _Quantize(x, 10.0)
x = x * 0.1
x = _Quantize(x, 1.0)
w = constant_op.constant(np.ones((8, 1)), dtype=dtypes.float32)
x = math_ops.matmul(x, w)
x = _Quantize(x, 10.0)
x = array_ops.identity(x, name=output_name)
return trt_test.TfTrtIntegrationTestParams(gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[[input_dims]],
output_names=[output_name],
expected_output_dims=[[[8, 1]]])
def GetParams(self):
"""Create a graph containing multiple segment."""
input_name = "input"
input_dims = [2, 32, 32, 3]
output_name = "output"
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(
dtype=dtypes.float32, shape=input_dims, name=input_name)
with g.device("/GPU:0"):
c1 = constant_op.constant(1.0, name="c1")
c2 = constant_op.constant(1.0, name="c2")
d1 = constant_op.constant(1.0, name="d1")
d2 = self.trt_incompatible_op(inp, name="d2")
with g.control_dependencies([d1, d2]):
add = math_ops.add(inp, c1, name="add")
with g.control_dependencies([d1, d2]):
mul = math_ops.mul(add, add, name="mul")
with g.control_dependencies([d1, d2]):
add1 = math_ops.add(mul, mul, name="add1")
edge = self.trt_incompatible_op(add1, name="incompatible")
with g.control_dependencies([d1, d2, add, mul]):
add2 = math_ops.add(edge, c2, name="add2")
with g.control_dependencies([d1, d2, add1, mul]):
mul1 = math_ops.mul(add2, add2, name="mul1")
with g.control_dependencies([d1, d2, add, add1]):
add3 = math_ops.add(mul1, mul1, name="add3")
array_ops.squeeze(add3, name=output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[tuple(input_dims)])
def GetParams(self):
"""Create a graph containing multiple segment."""
# TODO(aaroey): test graph with different dtypes.
dtype = dtypes.float32
input_name = "input"
input_dims = [100, 24, 24, 2]
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(dtype=dtype,
shape=[None] + input_dims[1:],
name=input_name)
with g.device("/GPU:0"):
conv_filter = constant_op.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]
],
name="weights",
dtype=dtype)
conv = nn.conv2d(input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
c1 = constant_op.constant(np.random.randn(
input_dims[0], 12, 12, 6),
dtype=dtype,
name="c1")
p = math_ops.mul(conv, c1, name="mul")
c2 = constant_op.constant(np.random.randn(
input_dims[0], 12, 12, 6),
dtype=dtype,
name="c2")
q = math_ops.div(conv, c2, name="div")
edge = self.trt_incompatible_op(q, name="incompatible")
edge = math_ops.div(edge, edge, name="div1")
r = math_ops.add(edge, edge, name="add")
p = math_ops.sub(p, edge, name="sub")
q = math_ops.mul(q, edge, name="mul1")
s = math_ops.add(p, q, name="add1")
s = math_ops.sub(s, r, name="sub1")
array_ops.squeeze(s, name=self.output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
# TODO(aaroey): LayoutOptimizer adds additional nodes to the graph which
# breaks the connection check, fix it.
# - my_trt_op_0 should have ["mul", "sub", "div1", "mul1", "add1",
# "add", "sub1"];
# - my_trt_op_1 should have ["weights","conv", "div"]
expected_engines=["my_trt_op_0", "my_trt_op_1"],
expected_output_dims=(100, 12, 12, 6),
allclose_atol=1.e-03,
allclose_rtol=1.e-03)
def GetParams(self):
dtype = dtypes.float32
input_name = "input"
input_dims = [100, 24, 24, 2]
output_name = "output"
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(dtype=dtype,
shape=[None] + input_dims[1:],
name=input_name)
outputs = []
# Here we test two types of reshapes, one changes the batch dimension and
# the other does not. Note that we're not able to test reshaping to
# scalar, since TRT requires input tensor to be of rank at least 2, so a
# reshape with scalar input will be filtered out of the segment before
# conversion.
with g.device("/GPU:0"):
# These reshapes happen at batch dimension, thus conversion should fail.
for shape in [[2, 50, 24, 24, 2], [-1, 50, 24, 24, 2],
[2, 50, -1, 24, 2]]:
incompatible_reshape = array_ops.reshape(inp, shape)
reshape_back = array_ops.reshape(incompatible_reshape,
[-1, 24, 24, 2])
outputs.append(self.trt_incompatible_op(reshape_back))
# Add another block with many reshapes that don't change the batch
# dimension.
compatible_reshape = array_ops.reshape(inp, [-1, 24 * 24, 2],
name="reshape-0")
compatible_reshape = array_ops.reshape(compatible_reshape,
[100, 24, -1],
name="reshape-1")
compatible_reshape = array_ops.reshape(compatible_reshape,
[100, 24 * 2, 24],
name="reshape-2")
compatible_reshape = array_ops.reshape(compatible_reshape,
[-1, 24, 24 * 2],
name="reshape-3")
compatible_reshape = array_ops.reshape(compatible_reshape,
[-1, 6, 4, 24, 2],
name="reshape-4")
compatible_reshape = array_ops.reshape(compatible_reshape,
[-1, 6, 4, 6, 4, 2, 1],
name="reshape-5")
compatible_reshape = array_ops.reshape(compatible_reshape,
[-1, 24, 24, 2],
name="reshape-6")
outputs.append(self.trt_incompatible_op(compatible_reshape))
math_ops.add_n(outputs, name=output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[tuple(input_dims)])
def GetParams(self):
"""Testing conversion of manually provided frozen graph."""
params_map = self._GetParamsMap()
gdef = graph_pb2.GraphDef()
with gfile.Open(params_map['graph_path'], 'rb') as f:
gdef.ParseFromString(f.read())
return trt_test.TfTrtIntegrationTestParams(
gdef=gdef,
input_names=params_map['input_names'],
input_dims=params_map['input_dims'],
output_names=params_map['output_names'],
expected_output_dims=params_map['expected_output_dims'])
def GetParams(self):
"""Test for multi connection neighboring nodes wiring tests in TF-TRT."""
dtype = dtypes.float32
input_name = "input"
input_dims = [2, 3, 7, 5]
output_name = "output"
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype,
shape=input_dims,
name=input_name)
e = constant_op.constant(np.random.normal(.05, .005, [3, 2, 3, 4]),
name="weights",
dtype=dtype)
conv = nn.conv2d(input=x,
filter=e,
data_format="NCHW",
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
b = constant_op.constant(np.random.normal(2.0, 1.0, [1, 4, 1, 1]),
name="bias",
dtype=dtype)
t = conv + b
b = constant_op.constant(np.random.normal(5.0, 1.0, [1, 4, 1, 1]),
name="bias",
dtype=dtype)
q = conv - b
edge = self.trt_incompatible_op(q)
b = constant_op.constant(np.random.normal(5.0, 1.0, [1, 4, 1, 1]),
name="bias",
dtype=dtype)
d = b + conv
edge3 = self.trt_incompatible_op(d)
edge1 = self.trt_incompatible_op(conv)
t = t - edge1
q = q + edge
t = t + q
t = t + d
t = t - edge3
array_ops.squeeze(t, name=output_name)
return trt_test.TfTrtIntegrationTestParams(gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[
(2, 4, 5, 4)
])
def GetParams(self):
"""Create a graph containing multiple segment."""
# TODO(aaroey): test graph with different dtypes.
dtype = dtypes.float32
input_name = "input"
input_dims = [100, 24, 24, 2]
output_name = "output"
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(dtype=dtype,
shape=[None] + input_dims[1:],
name=input_name)
with g.device("/GPU:0"):
conv_filter = constant_op.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]
],
name="weights",
dtype=dtype)
conv = nn.conv2d(input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
c1 = constant_op.constant(np.random.randn(
input_dims[0], 12, 12, 6),
dtype=dtype,
name="c1")
p = math_ops.mul(conv, c1, name="mul")
c2 = constant_op.constant(np.random.randn(
input_dims[0], 12, 12, 6),
dtype=dtype,
name="c2")
q = math_ops.div(conv, c2, name="div")
edge = self.trt_incompatible_op(q, name="incompatible")
edge = math_ops.div(edge, edge, name="div1")
r = math_ops.add(edge, edge, name="add")
p = math_ops.sub(p, edge, name="sub")
q = math_ops.mul(q, edge, name="mul1")
s = math_ops.add(p, q, name="add1")
s = math_ops.sub(s, r, name="sub1")
array_ops.squeeze(s, name=output_name)
return trt_test.TfTrtIntegrationTestParams(gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[
(100, 12, 12, 6)
])
def GetParams(self):
"""Testing conversion of BatchMatMul in TF-TRT conversion."""
dtype = dtypes.float32
input_name = "input"
input_dims = [12, 5, 8, 12]
output_name = "output"
w1_name = "matmul_w1"
w1_dims = [12, 5, 12, 7]
w2_name = "matmul_w2"
w2_dims = [12, 12, 7]
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(dtype=dtype,
shape=[None] + input_dims[1:],
name=input_name)
w1 = array_ops.placeholder(dtype=dtype,
shape=w1_dims,
name=w1_name)
w2 = array_ops.placeholder(dtype=dtype,
shape=w2_dims,
name=w2_name)
with g.device("/GPU:0"):
b = constant_op.constant(np.random.randn(12, 5, 12, 7),
dtype=dtype)
x1 = math_ops.matmul(inp, b)
c = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtype)
x1 = x1 + c
x2 = math_ops.matmul(inp, w1)
d = constant_op.constant(np.random.randn(5, 1, 1), dtype=dtype)
x2 = x2 * d
e = self.trt_incompatible_op(inp)
e = gen_array_ops.reshape(e, [12, 40, 12])
x3 = math_ops.matmul(e, w2)
f = constant_op.constant(np.random.randn(40, 1), dtype=dtype)
x3 = x3 + f
x3 = gen_array_ops.reshape(x3, [12, 5, 8, 7])
x3 = self.trt_incompatible_op(x3)
out = x1 + x2 + x3
array_ops.squeeze(out, name=output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name, w1_name, w2_name],
input_dims=[[input_dims, w1_dims, w2_dims]],
output_names=[output_name],
expected_output_dims=[[[12, 5, 8, 7]]])
def GetParams(self):
"""Create a graph containing multiple segment."""
# TODO(aaroey): test graph with different dtypes.
dtype = dtypes.float32
input_name = "input"
input_dims = [100, 24, 24, 2]
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(dtype=dtype,
shape=[None] + input_dims[1:],
name=input_name)
with g.device("/GPU:0"):
conv_filter = constant_op.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]
],
name="weights",
dtype=dtype)
conv = nn.conv2d(input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
c1 = constant_op.constant(np.random.randn(
input_dims[0], 12, 12, 6),
dtype=dtype)
p = conv * c1
c2 = constant_op.constant(np.random.randn(
input_dims[0], 12, 12, 6),
dtype=dtype)
q = conv / c2
edge = self.trt_incompatible_op(q)
edge /= edge
r = edge + edge
p -= edge
q *= edge
s = p + q
s -= r
array_ops.squeeze(s, name=self.output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
num_expected_engines=2,
expected_output_dims=(100, 12, 12, 6),
allclose_atol=1.e-03,
allclose_rtol=1.e-03)
def GetParams(self):
# TODO(laigd): we should test the following cases:
# - batch size is not changed, other dims are changing
# - batch size is decreasing, other dims are identical
# - batch size is decreasing, other dims are changing
# - batch size is increasing, other dims are identical
# - batch size is increasing, other dims are changing
input_dims = [[[1, 5, 5, 1]], [[10, 5, 5, 1]], [[3, 5, 5, 1]],
[[1, 5, 5, 1]], [[1, 3, 1, 1]], [[2, 9, 9, 1]],
[[1, 224, 224, 1]], [[1, 128, 224, 1]]]
expected_output_dims = input_dims
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(shape=(None, None, None, 1),
dtype=dtypes.float32,
name="input")
conv_filter1 = constant_op.constant(np.ones([3, 3, 1, 8]),
name="weights1",
dtype=dtypes.float32)
bias1 = constant_op.constant(np.random.randn(8),
dtype=dtypes.float32)
x = nn.conv2d(input=x,
filter=conv_filter1,
strides=[1, 1, 1, 1],
padding="SAME",
name="conv")
x = nn.bias_add(x, bias1)
x = nn.relu(x)
conv_filter2 = constant_op.constant(np.ones([3, 3, 8, 1]),
name="weights2",
dtype=dtypes.float32)
bias2 = constant_op.constant(np.random.randn(1),
dtype=dtypes.float32)
x = nn.conv2d(input=x,
filter=conv_filter2,
strides=[1, 1, 1, 1],
padding="SAME",
name="conv")
x = nn.bias_add(x, bias2)
x = array_ops.identity(x, name="output")
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=["input"],
input_dims=input_dims,
output_names=["output"],
expected_output_dims=expected_output_dims)
def GetParams(self):
"""Testing conversion of Conv2D (data_format=NCHW) in TF-TRT conversion."""
np.random.seed(1234)
input_dims = [13, 3, 7, 11]
g = build_graph(input_dims=input_dims,
dtype=dtypes.float32,
num_filters=5,
data_format="channels_first",
kernel_sizes=[(3, 3), (3, 2)],
dilation_rates=[(1, 1), (2, 3)])
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=["input"],
input_dims=[[input_dims]],
output_names=["output"],
expected_output_dims=[[[13, 5, 7, 11]]])
def GetParams(self):
"""Testing Concatenation in TF-TRT conversion."""
dtype = dtypes.float32
input_name = "input"
input_dims = [2, 3, 3, 1]
output_name = "output"
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype,
shape=input_dims,
name=input_name)
# scale
a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtype)
r1 = x / a
a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtype)
r2 = a / x
a = constant_op.constant(np.random.randn(1, 3, 1), dtype=dtype)
r3 = a + x
a = constant_op.constant(np.random.randn(1, 3, 1), dtype=dtype)
r4 = x * a
a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtype)
r5 = x - a
a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtype)
r6 = a - x
a = constant_op.constant(np.random.randn(3, 1), dtype=dtype)
r7 = x - a
a = constant_op.constant(np.random.randn(3, 1), dtype=dtype)
r8 = a - x
a = constant_op.constant(np.random.randn(3, 1, 1), dtype=dtype)
r9 = gen_math_ops.maximum(x, a)
a = constant_op.constant(np.random.randn(3, 1), dtype=dtype)
r10 = gen_math_ops.minimum(a, x)
a = constant_op.constant(np.random.randn(3), dtype=dtype)
r11 = x * a
a = constant_op.constant(np.random.randn(1), dtype=dtype)
r12 = a * x
concat1 = array_ops.concat([r1, r2, r3, r4, r5, r6], axis=-1)
concat2 = array_ops.concat([r7, r8, r9, r10, r11, r12], axis=3)
x = array_ops.concat([concat1, concat2], axis=-1)
gen_array_ops.reshape(x, [2, -1], name=output_name)
return trt_test.TfTrtIntegrationTestParams(gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[
(2, 126)
])
def GetParams(self):
"""Single vgg layer in NCHW unit tests in TF-TRT."""
dtype = dtypes.float32
input_name = "input"
input_dims = [5, 2, 8, 8]
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype,
shape=input_dims,
name=input_name)
x, _, _ = nn_impl.fused_batch_norm(
x,
np.random.randn(2).astype(np.float32),
np.random.randn(2).astype(np.float32),
mean=np.random.randn(2).astype(np.float32),
variance=np.random.randn(2).astype(np.float32),
data_format="NCHW",
is_training=False)
e = constant_op.constant(np.random.randn(1, 1, 2, 6),
name="weights",
dtype=dtype)
conv = nn.conv2d(input=x,
filter=e,
data_format="NCHW",
strides=[1, 1, 2, 2],
padding="SAME",
name="conv")
b = constant_op.constant(np.random.randn(6),
name="bias",
dtype=dtype)
t = nn.bias_add(conv, b, data_format="NCHW", name="biasAdd")
relu = nn.relu(t, "relu")
idty = array_ops.identity(relu, "ID")
v = nn_ops.max_pool(idty, [1, 1, 2, 2], [1, 1, 2, 2],
"VALID",
data_format="NCHW",
name="max_pool")
array_ops.squeeze(v, name="output")
return trt_test.TfTrtIntegrationTestParams(gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
num_expected_engines=1,
expected_output_dims=(5, 6,
2, 2),
allclose_atol=1.e-03,
allclose_rtol=1.e-03)
def GetParams(self):
"""Create a graph containing single segment."""
# TODO(aaroey): test graph with different dtypes.
dtype = dtypes.float32
input_name = "input"
input_dims = [100, 24, 24, 2]
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(dtype=dtype,
shape=[None] + input_dims[1:],
name=input_name)
with g.device("/GPU:0"):
conv_filter = constant_op.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]
],
name="weights",
dtype=dtype)
conv = nn.conv2d(input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
bias = constant_op.constant([4., 1.5, 2., 3., 5., 7.],
name="bias",
dtype=dtype)
added = nn.bias_add(conv, bias, name="bias_add")
relu = nn.relu(added, "relu")
identity = array_ops.identity(relu, "identity")
pool = nn_ops.max_pool(identity, [1, 2, 2, 1], [1, 2, 2, 1],
"VALID",
name="max_pool")
array_ops.squeeze(pool, name=self.output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
# TODO(aaroey): LayoutOptimizer adds additional nodes to the graph which
# breaks the connection check, fix it.
# - my_trt_op_0 should have ["weights", "conv", "bias", "bias_add",
# "relu", "identity", "max_pool"]
expected_engines=["my_trt_op_0"],
expected_output_dims=(100, 6, 6, 6),
allclose_atol=1.e-03,
allclose_rtol=1.e-03)
def GetParams(self):
"""Create a graph containing single segment."""
dtype = dtypes.float32
input_name = "input"
input_dims = [100, 24, 24, 2]
output_name = "output"
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(dtype=dtype,
shape=[None] + input_dims[1:],
name=input_name)
with g.device("/GPU:0"):
# Add a block with compatible transposes.
compatible_transpose = array_ops.transpose(inp, [0, 3, 1, 2],
name="transpose-1")
compatible_transpose = array_ops.transpose(
compatible_transpose, [0, 2, 3, 1], name="transposeback")
# Add an incompatible op so the first block will not be in the same
# subgraph where the following block belongs.
bridge = self.trt_incompatible_op(compatible_transpose)
# Add a block with incompatible transposes.
#
# Note: by default Grappler will run the TRT optimizer twice. At the
# first time it will group the two transpose ops below to same segment
# then fail the conversion due to the expected batch dimension problem.
# At the second time, since the input of bridge op is TRTEngineOp_0, it
# will fail to do shape inference which then cause conversion to fail.
# TODO(laigd): support shape inference, make TRT optimizer run only
# once, and fix this.
incompatible_transpose = array_ops.transpose(
bridge, [2, 1, 0, 3], name="transpose-2")
excluded_transpose = array_ops.transpose(
incompatible_transpose, [0, 2, 3, 1], name="transpose-3")
array_ops.identity(excluded_transpose, name=output_name)
return trt_test.TfTrtIntegrationTestParams(gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[
(24, 100, 2, 24)
])
