def GetParams(self):
"""Testing conversion of BatchMatMul in TF-TRT conversion."""
dtype = dtypes.float32
input_name = "input"
input_dims = [2, 15, 15, 3]
output_name = "output"
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(dtype=dtype,
shape=[None] + input_dims[1:],
name=input_name)
with g.device("/GPU:0"):
e1 = constant_op.constant(np.random.randn(1, 1, 3, 5),
name="kernel_1",
dtype=dtype)
e2 = constant_op.constant(np.random.randn(1, 1, 5, 10),
name="kernel_2",
dtype=dtype)
conv = nn.conv2d(input=inp,
filter=e1,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
out = nn.conv2d(input=conv,
filter=e2,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv_2")
array_ops.squeeze(out, name=output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[[input_dims]],
output_names=[output_name],
expected_output_dims=[[[2, 15, 15, 10]]])
def GetParams(self):
"""Test for Constant broadcasting in TF-TRT."""
dtype = dtypes.float32
input_name = 'input'
input_dims = [5, 12, 12, 2]
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
filt1 = constant_op.constant(
0.3, shape=(3, 3, 2, 1), dtype=dtype, name='filt1')
y1 = nn.conv2d(x, filt1, strides=[1, 1, 1, 1], padding='SAME', name='y1')
z1 = nn.relu(y1, name='z1')
filt2 = constant_op.constant(
np.random.randn(9), shape=(3, 3, 1, 1), dtype=dtype, name='filt2')
y2 = nn.conv2d(z1, filt2, strides=[1, 1, 1, 1], padding='SAME', name='y2')
z2 = nn.relu(y2, name='z')
filt3 = constant_op.constant(
np.random.randn(3, 3, 1, 1),
shape=(3, 3, 1, 1),
dtype=dtype,
name='filt3')
y3 = nn.conv2d(z2, filt3, strides=[1, 1, 1, 1], padding='SAME', name='y3')
nn.relu(y3, name='output')
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
num_expected_engines=1,
expected_output_dims=(5, 12, 12, 1),
allclose_atol=1.e-02,
allclose_rtol=1.e-02)
def GraphFn(self, x):
"""Return the expected graph to convert."""
dtype = x.dtype
filt1 = constant_op.constant(0.3,
shape=(3, 3, 2, 1),
dtype=dtype,
name='filt1')
y1 = nn.conv2d(x,
filt1,
strides=[1, 1, 1, 1],
padding='SAME',
name='y1')
z1 = nn.relu(y1, name='z1')
filt2 = constant_op.constant(0.3,
shape=(3, 3, 1, 1),
dtype=dtype,
name='filt2')
y2 = nn.conv2d(z1,
filt2,
strides=[1, 1, 1, 1],
padding='SAME',
name='y2')
z2 = nn.relu(y2, name='z')
filt3 = constant_op.constant(0.3,
shape=(3, 3, 1, 1),
dtype=dtype,
name='filt3')
y3 = nn.conv2d(z2,
filt3,
strides=[1, 1, 1, 1],
padding='SAME',
name='y3')
return nn.relu(y3, name='output_0')
def GetParams(self):
"""Testing conversion of BatchMatMul in TF-TRT conversion."""
dtype = dtypes.float32
input_name = "input"
input_dims = [2, 15, 15, 3]
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(
dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
with g.device("/GPU:0"):
e1 = constant_op.constant(
np.random.randn(1, 1, 3, 5), name="kernel_1", dtype=dtype)
e2 = constant_op.constant(
np.random.randn(1, 1, 5, 10), name="kernel_2", dtype=dtype)
conv = nn.conv2d(
input=inp,
filter=e1,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
out = nn.conv2d(
input=conv,
filter=e2,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv_2")
array_ops.squeeze(out, name=self.output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
expected_engines=["my_trt_op_0"],
expected_output_dims=(2, 15, 15, 10),
allclose_atol=1.e-02,
allclose_rtol=1.e-02)
def GraphFn(self, x):
dtype = x.dtype
filt1 = constant_op.constant(0.3,
shape=(3, 3, 2, 1),
dtype=dtype,
name='filt1')
y1 = nn.conv2d(x,
filt1,
strides=[1, 1, 1, 1],
padding='SAME',
name='y1')
z1 = nn.relu(y1, name='z1')
filt2 = constant_op.constant(np.random.randn(9),
shape=(3, 3, 1, 1),
dtype=dtype,
name='filt2')
y2 = nn.conv2d(z1,
filt2,
strides=[1, 1, 1, 1],
padding='SAME',
name='y2')
z2 = nn.relu(y2, name='z')
filt3 = constant_op.constant(np.random.randn(3, 3, 1, 1),
shape=(3, 3, 1, 1),
dtype=dtype,
name='filt3')
y3 = nn.conv2d(z2,
filt3,
strides=[1, 1, 1, 1],
padding='SAME',
name='y3')
return nn.relu(y3, name='output_0')
def GetParams(self):
# TODO(laigd): we should test the following cases:
# - batch size is not changed, other dims are changing
# - batch size is decreasing, other dims are identical
# - batch size is decreasing, other dims are changing
# - batch size is increasing, other dims are identical
# - batch size is increasing, other dims are changing
input_dims = [[[1, 5, 5, 1]], [[10, 5, 5, 1]], [[3, 5, 5, 1]],
[[1, 5, 5, 1]], [[1, 3, 1, 1]], [[2, 9, 9, 1]],
[[1, 224, 224, 1]], [[1, 128, 224, 1]]]
expected_output_dims = input_dims
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(shape=(None, None, None, 1),
dtype=dtypes.float32,
name="input")
conv_filter1 = constant_op.constant(np.ones([3, 3, 1, 8]),
name="weights1",
dtype=dtypes.float32)
bias1 = constant_op.constant(np.random.randn(8),
dtype=dtypes.float32)
x = nn.conv2d(input=x,
filter=conv_filter1,
strides=[1, 1, 1, 1],
padding="SAME",
name="conv")
x = nn.bias_add(x, bias1)
x = nn.relu(x)
conv_filter2 = constant_op.constant(np.ones([3, 3, 8, 1]),
name="weights2",
dtype=dtypes.float32)
bias2 = constant_op.constant(np.random.randn(1),
dtype=dtypes.float32)
x = nn.conv2d(input=x,
filter=conv_filter2,
strides=[1, 1, 1, 1],
padding="SAME",
name="conv")
x = nn.bias_add(x, bias2)
x = array_ops.identity(x, name="output")
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=["input"],
input_dims=input_dims,
output_names=["output"],
expected_output_dims=expected_output_dims)
def GetParams(self):
"""Neighboring node wiring tests in TF-TRT conversion."""
dtype = dtypes.float32
input_name = "input"
input_dims = [2, 3, 7, 5]
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
e = constant_op.constant(
np.random.normal(.3, 0.05, [3, 2, 3, 4]), name="weights", dtype=dtype)
conv = nn.conv2d(
input=x,
filter=e,
data_format="NCHW",
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
b = constant_op.constant(
np.random.normal(1.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
t = conv * b
e = gen_math_ops.tan(conv)
t = t - e
array_ops.squeeze(t, name=self.output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
num_expected_engines=2,
expected_output_dims=(2, 4, 5, 4),
allclose_atol=1.e-03,
allclose_rtol=1.e-03)
def _annotated_graph(self):
graph = ops.Graph()
with graph.as_default():
random_seed.set_random_seed(2)
current_activation = variable_scope.get_variable(
name='start', shape=[1, 2, 2, 5])
conv_filter = variable_scope.get_variable(name='filter',
shape=[5, 5, 5, 5])
for layer_number in range(3):
with variable_scope.variable_scope(
'layer_{}'.format(layer_number)):
after_conv = nn.conv2d(current_activation, conv_filter,
[1, 1, 1, 1], 'SAME')
current_activation = 2. * after_conv
current_activation.op._set_attr(
'_recompute_hint',
# The value of the attribute does not matter; just that the key
# exists in the op's attributes.
attr_value_pb2.AttrValue(i=1))
current_activation += 5.
current_activation.op._set_attr(
'_recompute_hint', attr_value_pb2.AttrValue(i=0))
current_activation = nn.relu(current_activation)
current_activation.op._set_attr(
'_recompute_hint', attr_value_pb2.AttrValue(i=1))
loss = math_ops.reduce_mean(current_activation)
optimizer = train.AdamOptimizer(0.001)
train_op = optimizer.minimize(loss)
init_op = variables.global_variables_initializer()
return graph, init_op, train_op
def GraphFn(self, x):
dtype = x.dtype
x, _, _ = nn_impl.fused_batch_norm(
x, [1.0, 1.0], [0.0, 0.0],
mean=[0.5, 0.5],
variance=[1.0, 1.0],
data_format="NCHW",
is_training=False)
e = constant_op.constant(
np.random.randn(1, 1, 2, 6), name="weights", dtype=dtype)
conv = nn.conv2d(
input=x,
filter=e,
data_format="NCHW",
strides=[1, 1, 2, 2],
padding="SAME",
name="conv")
b = constant_op.constant(np.random.randn(6), name="bias", dtype=dtype)
t = nn.bias_add(conv, b, data_format="NCHW", name="biasAdd")
relu = nn.relu(t, "relu")
idty = array_ops.identity(relu, "ID")
v = nn_ops.max_pool(
idty, [1, 1, 2, 2], [1, 1, 2, 2],
"VALID",
data_format="NCHW",
name="max_pool")
return array_ops.squeeze(v, name="output_0")
def GetParams(self):
"""Neighboring node wiring tests in TF-TRT conversion."""
dtype = dtypes.float32
input_name = "input"
input_dims = [2, 3, 7, 5]
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype,
shape=input_dims,
name=input_name)
e = constant_op.constant(np.random.normal(.3, 0.05, [3, 2, 3, 4]),
name="weights",
dtype=dtype)
conv = nn.conv2d(input=x,
filter=e,
data_format="NCHW",
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
b = constant_op.constant(np.random.normal(1.0, 1.0, [1, 4, 1, 1]),
name="bias",
dtype=dtype)
t = conv * b
e = gen_math_ops.tan(conv)
t = t - e
array_ops.squeeze(t, name=self.output_name)
return trt_test.TfTrtIntegrationTestParams(gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
num_expected_engines=2,
expected_output_dims=(2, 4,
5, 4),
allclose_atol=1.e-03,
allclose_rtol=1.e-03)
def get_simple_graph_def(self):
"""Create a simple graph and return its graph_def."""
g = ops.Graph()
with g.as_default():
a = aops.placeholder(
dtype=dtypes.float32, shape=(None, 24, 24, 2), name="input")
e = cop.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
name="weights",
dtype=dtypes.float32)
conv = nn.conv2d(
input=a,
filter=e,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
b = cop.constant(
[4., 1.5, 2., 3., 5., 7.], name="bias", dtype=dtypes.float32)
t = nn.bias_add(conv, b, name="biasAdd")
relu = nn.relu(t, "relu")
idty = aops.identity(relu, "ID")
v = nn_ops.max_pool(
idty, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool")
aops.squeeze(v, name="output")
return g.as_graph_def()
def GraphFn(self, inp):
"""Create a graph containing multiple segment."""
dtype = inp.dtype
conv_filter = constant_op.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
name="weights",
dtype=dtype)
conv = nn.conv2d(input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
c1 = constant_op.constant(np.random.randn(12, 12, 6),
dtype=dtype,
name="c1")
p = math_ops.mul(conv, c1, name="mul")
c2 = constant_op.constant(np.random.randn(12, 12, 6),
dtype=dtype,
name="c2")
q = math_ops.div(conv, c2, name="div")
edge = self.trt_incompatible_op(q, name="incompatible")
edge = math_ops.div(edge, edge, name="div1")
r = math_ops.add(edge, edge, name="add")
p = math_ops.sub(p, edge, name="sub")
q = math_ops.mul(q, edge, name="mul1")
s = math_ops.add(p, q, name="add1")
s = math_ops.sub(s, r, name="sub1")
return array_ops.squeeze(s, name="output_0")
def GetParams(self):
"""Neighboring node wiring tests in TF-TRT conversion."""
dtype = dtypes.float32
input_name = "input"
input_dims = [2, 3, 7, 5]
output_name = "output"
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
e = constant_op.constant(
np.random.normal(.3, 0.05, [3, 2, 3, 4]), name="weights", dtype=dtype)
conv = nn.conv2d(
input=x,
filter=e,
data_format="NCHW",
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
b = constant_op.constant(
np.random.normal(1.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
t = math_ops.mul(conv, b, name="mul")
e = self.trt_incompatible_op(conv, name="incompatible")
t = math_ops.sub(t, e, name="sub")
array_ops.squeeze(t, name=output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[(2, 4, 5, 4)])
def _annotated_graph(self):
graph = ops.Graph()
with graph.as_default():
random_seed.set_random_seed(2)
current_activation = variable_scope.get_variable(
name='start', shape=[1, 2, 2, 5])
conv_filter = variable_scope.get_variable(
name='filter', shape=[5, 5, 5, 5])
for layer_number in range(3):
with variable_scope.variable_scope('layer_{}'.format(layer_number)):
after_conv = nn.conv2d(current_activation, conv_filter, [1, 1, 1, 1],
'SAME')
current_activation = 2. * after_conv
current_activation.op._set_attr(
'_recompute_hint',
# The value of the attribute does not matter; just that the key
# exists in the op's attributes.
attr_value_pb2.AttrValue(i=1))
current_activation += 5.
current_activation.op._set_attr(
'_recompute_hint', attr_value_pb2.AttrValue(i=0))
current_activation = nn.relu(current_activation)
current_activation.op._set_attr(
'_recompute_hint', attr_value_pb2.AttrValue(i=1))
loss = math_ops.reduce_mean(current_activation)
optimizer = train.AdamOptimizer(0.001)
train_op = optimizer.minimize(loss)
init_op = variables.global_variables_initializer()
return graph, init_op, train_op
def GetSingleEngineGraphDef(dtype=dtypes.float32):
"""Create a graph containing single segment."""
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(dtype=dtype,
shape=[None] + INPUT_DIMS[1:],
name=INPUT_NAME)
with g.device("/GPU:0"):
conv_filter = constant_op.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
name="weights",
dtype=dtype)
conv = nn.conv2d(input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
bias = constant_op.constant([4., 1.5, 2., 3., 5., 7.],
name="bias",
dtype=dtype)
added = nn.bias_add(conv, bias, name="bias_add")
relu = nn.relu(added, "relu")
identity = array_ops.identity(relu, "identity")
pool = nn_ops.max_pool(identity, [1, 2, 2, 1], [1, 2, 2, 1],
"VALID",
name="max_pool")
array_ops.squeeze(pool, name=OUTPUT_NAME)
return g.as_graph_def()
def GetParams(self):
dtype = dtypes.float32
input_name = "input"
input_dims = [[[1, 10, 10, 2]], [[2, 10, 10, 2]], [[4, 10, 10, 2]],
[[2, 10, 10, 2]]]
expected_output_dims = [[[1, 10, 10, 1]], [[2, 10, 10, 1]], [[4, 10, 10,
1]],
[[2, 10, 10, 1]]]
output_name = "output"
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(
dtype=dtype, shape=[None, 10, 10, 2], name=input_name)
conv_filter = constant_op.constant(
np.random.randn(3, 3, 2, 1), dtype=dtypes.float32)
x = nn.conv2d(
input=x,
filter=conv_filter,
strides=[1, 1, 1, 1],
padding="SAME",
name="conv")
bias = constant_op.constant(
np.random.randn(1, 10, 10, 1), dtype=dtypes.float32)
x = math_ops.add(x, bias)
x = nn.relu(x)
x = array_ops.identity(x, name="output")
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=input_dims,
output_names=[output_name],
expected_output_dims=expected_output_dims)
def _GetMetaGraph(self,
batch_size=14,
image_dim=12,
optimizer_scope_name=''):
"""A simple layered graph with conv, an intermediate op, and a ReLU."""
graph = ops.Graph()
with graph.as_default():
random_seed.set_random_seed(1)
current_activation = variable_scope.get_variable(
name='start', shape=[batch_size, image_dim, image_dim, 5])
conv_filter = variable_scope.get_variable(name='filter',
shape=[5, 5, 5, 5])
for layer_number in range(10):
with variable_scope.variable_scope(
'layer_{}'.format(layer_number)):
after_conv = nn.conv2d(current_activation, conv_filter,
[1, 1, 1, 1], 'SAME')
current_activation = 2. * after_conv
current_activation = nn.relu(current_activation)
loss = math_ops.reduce_mean(current_activation)
with ops.name_scope(optimizer_scope_name):
optimizer = train.AdamOptimizer(0.001)
train_op = optimizer.minimize(loss)
init_op = variables.global_variables_initializer()
metagraph = train.export_meta_graph()
return (metagraph, init_op.name, train_op.name, loss.name)
def GetSingleEngineGraphDef(dtype=dtypes.float32):
"""Create a graph containing single segment."""
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(
dtype=dtype, shape=[None] + INPUT_DIMS[1:], name=INPUT_NAME)
with g.device("/GPU:0"):
conv_filter = constant_op.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
name="weights",
dtype=dtype)
conv = nn.conv2d(
input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
bias = constant_op.constant(
[4., 1.5, 2., 3., 5., 7.], name="bias", dtype=dtype)
added = nn.bias_add(conv, bias, name="bias_add")
relu = nn.relu(added, "relu")
identity = array_ops.identity(relu, "identity")
pool = nn_ops.max_pool(
identity, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool")
array_ops.squeeze(pool, name=OUTPUT_NAME)
return g.as_graph_def()
def GetMultiEngineGraphDef(dtype=dtypes.float32):
"""Create a graph containing multiple segment."""
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(
dtype=dtype, shape=[None] + INPUT_DIMS[1:], name=INPUT_NAME)
with g.device("/GPU:0"):
conv_filter = constant_op.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
name="weights",
dtype=dtype)
conv = nn.conv2d(
input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
c1 = constant_op.constant(
np.random.randn(INPUT_DIMS[0], 12, 12, 6), dtype=dtype)
p = conv * c1
c2 = constant_op.constant(
np.random.randn(INPUT_DIMS[0], 12, 12, 6), dtype=dtype)
q = conv / c2
edge = math_ops.sin(q)
edge /= edge
r = edge + edge
p -= edge
q *= edge
s = p + q
s -= r
array_ops.squeeze(s, name=OUTPUT_NAME)
return g.as_graph_def()
def GetParams(self):
"""Single vgg layer test in TF-TRT conversion."""
dtype = dtypes.float32
input_name = "input"
input_dims = [5, 8, 8, 2]
output_name = "output"
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
x, _, _ = nn_impl.fused_batch_norm(
x, [1.0, 1.0], [0.0, 0.0],
mean=[0.5, 0.5],
variance=[1.0, 1.0],
is_training=False)
e = constant_op.constant(
np.random.randn(1, 1, 2, 6), name="weights", dtype=dtype)
conv = nn.conv2d(
input=x, filter=e, strides=[1, 2, 2, 1], padding="SAME", name="conv")
b = constant_op.constant(np.random.randn(6), name="bias", dtype=dtype)
t = nn.bias_add(conv, b, name="biasAdd")
relu = nn.relu(t, "relu")
idty = array_ops.identity(relu, "ID")
v = nn_ops.max_pool(
idty, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool")
array_ops.squeeze(v, name=output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[(5, 2, 2, 6)])
def GetParams(self):
dtype = dtypes.float32
input_name = "input"
input_dims = [[[1, 10, 10, 2]], [[2, 10, 10, 2]], [[4, 10, 10, 2]],
[[2, 10, 10, 2]]]
expected_output_dims = [[[1, 10, 10, 1]], [[2, 10, 10, 1]],
[[4, 10, 10, 1]], [[2, 10, 10, 1]]]
output_name = "output"
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype,
shape=[None, 10, 10, 2],
name=input_name)
conv_filter = constant_op.constant(np.random.randn(3, 3, 2, 1),
dtype=dtypes.float32)
x = nn.conv2d(input=x,
filter=conv_filter,
strides=[1, 1, 1, 1],
padding="SAME",
name="conv")
bias = constant_op.constant(np.random.randn(1, 10, 10, 1),
dtype=dtypes.float32)
x = math_ops.add(x, bias)
x = nn.relu(x)
x = array_ops.identity(x, name="output")
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=input_dims,
output_names=[output_name],
expected_output_dims=expected_output_dims)
def get_simple_graph_def():
"""Create a simple graph and return its graph_def."""
g = ops.Graph()
with g.as_default():
a = aops.placeholder(dtype=dtypes.float32,
shape=(None, 24, 24, 2),
name="input")
e = cop.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
name="weights",
dtype=dtypes.float32)
conv = nn.conv2d(input=a,
filter=e,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
b = cop.constant([4., 1.5, 2., 3., 5., 7.],
name="bias",
dtype=dtypes.float32)
t = nn.bias_add(conv, b, name="biasAdd")
relu = nn.relu(t, "relu")
idty = aops.identity(relu, "ID")
v = nn_ops.max_pool(idty, [1, 2, 2, 1], [1, 2, 2, 1],
"VALID",
name="max_pool")
aops.squeeze(v, name="output")
return g.as_graph_def()
def GetParams(self):
"""Neighboring node wiring tests in TF-TRT conversion."""
dtype = dtypes.float32
input_name = "input"
input_dims = [2, 3, 7, 5]
output_name = "output"
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype,
shape=input_dims,
name=input_name)
e = constant_op.constant(np.random.normal(.3, 0.05, [3, 2, 3, 4]),
name="weights",
dtype=dtype)
conv = nn.conv2d(input=x,
filter=e,
data_format="NCHW",
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
b = constant_op.constant(np.random.normal(1.0, 1.0, [1, 4, 1, 1]),
name="bias",
dtype=dtype)
t = math_ops.mul(conv, b, name="mul")
e = self.trt_incompatible_op(conv, name="incompatible")
t = math_ops.sub(t, e, name="sub")
array_ops.squeeze(t, name=output_name)
return trt_test.TfTrtIntegrationTestParams(gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[
(2, 4, 5, 4)
])
def _RunGraphWithConfig(self, config, batch_size=14, image_dim=12):
"""Run a simple layered graph with conv, an intermediate op, and a ReLU."""
graph = ops.Graph()
with graph.as_default():
random_seed.set_random_seed(1)
current_activation = variable_scope.get_variable(
name='start', shape=[batch_size, image_dim, image_dim, 5])
conv_filter = variable_scope.get_variable(name='filter',
shape=[5, 5, 5, 5])
for layer_number in range(10):
with variable_scope.variable_scope(
'layer_{}'.format(layer_number)):
after_conv = nn.conv2d(current_activation, conv_filter,
[1, 1, 1, 1], 'SAME')
current_activation = 2. * after_conv
current_activation = nn.relu(current_activation)
loss = math_ops.reduce_mean(current_activation)
optimizer = train.AdamOptimizer(0.001)
train_op = optimizer.minimize(loss)
init_op = variables.global_variables_initializer()
with session.Session(config=config, graph=graph) as sess:
sess.run(init_op)
sess.run(train_op)
sess.run(train_op)
return sess.run(loss)
def GraphFn(self, x):
dtype = x.dtype
e = constant_op.constant(
np.random.normal(.05, .005, [3, 2, 3, 4]), name="weights", dtype=dtype)
conv = nn.conv2d(
input=x,
filter=e,
data_format="NCHW",
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
b = constant_op.constant(
np.random.normal(2.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
t = conv + b
b = constant_op.constant(
np.random.normal(5.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
q = conv - b
edge = self.trt_incompatible_op(q)
b = constant_op.constant(
np.random.normal(5.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
d = b + conv
edge3 = self.trt_incompatible_op(d)
edge1 = self.trt_incompatible_op(conv)
t = t - edge1
q = q + edge
t = t + q
t = t + d
t = t - edge3
return array_ops.squeeze(t, name="output_0")
def loop_fn(i):
x1 = array_ops.gather(x, i)
return nn.conv2d(x1,
filt,
strides=[1, 2, 2, 1],
padding="VALID",
data_format="NHWC")
def model(device):
with ops.device(device):
x = array_ops.placeholder(np.float32, shape=[2])
x_bcast = gen_array_ops.broadcast_to(
x, shape=[2, 256, 256, 2])
w_bcast = gen_array_ops.broadcast_to(x, shape=[2, 2, 2, 2])
y = nn.conv2d(x_bcast,
w_bcast,
strides=1,
padding="SAME",
name="a")
y = nn.conv2d(y,
w_bcast,
strides=1,
padding="SAME",
name="b")
return sess.run(y, {x: np.ones(x.shape)})
def GetParams(self):
# TODO(laigd): we should test the following cases:
# - batch size is not changed, other dims are changing
# - batch size is decreasing, other dims are identical
# - batch size is decreasing, other dims are changing
# - batch size is increasing, other dims are identical
# - batch size is increasing, other dims are changing
input_dims = [[[1, 5, 5, 1]], [[10, 5, 5, 1]], [[3, 5, 5, 1]],
[[1, 5, 5, 1]], [[1, 3, 1, 1]], [[2, 9, 9, 1]],
[[1, 224, 224, 1]], [[1, 128, 224, 1]]]
expected_output_dims = input_dims
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(
shape=(None, None, None, 1), dtype=dtypes.float32, name="input")
conv_filter1 = constant_op.constant(
np.ones([3, 3, 1, 8]), name="weights1", dtype=dtypes.float32)
bias1 = constant_op.constant(np.random.randn(8), dtype=dtypes.float32)
x = nn.conv2d(
input=x,
filter=conv_filter1,
strides=[1, 1, 1, 1],
padding="SAME",
name="conv")
x = nn.bias_add(x, bias1)
x = nn.relu(x)
conv_filter2 = constant_op.constant(
np.ones([3, 3, 8, 1]), name="weights2", dtype=dtypes.float32)
bias2 = constant_op.constant(np.random.randn(1), dtype=dtypes.float32)
x = nn.conv2d(
input=x,
filter=conv_filter2,
strides=[1, 1, 1, 1],
padding="SAME",
name="conv")
x = nn.bias_add(x, bias2)
x = array_ops.identity(x, name="output")
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=["input"],
input_dims=input_dims,
output_names=["output"],
expected_output_dims=expected_output_dims)
def GraphFn(self, inp):
dtype = inp.dtype
e1 = constant_op.constant(np.random.randn(1, 1, 3, 5),
name="kernel_1",
dtype=dtype)
e2 = constant_op.constant(np.random.randn(1, 1, 5, 10),
name="kernel_2",
dtype=dtype)
conv = nn.conv2d(input=inp,
filter=e1,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
out = nn.conv2d(input=conv,
filter=e2,
strides=[1, 1, 1, 1],
padding="VALID",
name="conv_2")
return array_ops.squeeze(out, name="output_0")
def GetParams(self):
"""Create a graph containing multiple segment."""
# TODO(aaroey): test graph with different dtypes.
dtype = dtypes.float32
input_name = "input"
input_dims = [100, 24, 24, 2]
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(dtype=dtype,
shape=[None] + input_dims[1:],
name=input_name)
with g.device("/GPU:0"):
conv_filter = constant_op.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]
],
name="weights",
dtype=dtype)
conv = nn.conv2d(input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
c1 = constant_op.constant(np.random.randn(
input_dims[0], 12, 12, 6),
dtype=dtype,
name="c1")
p = math_ops.mul(conv, c1, name="mul")
c2 = constant_op.constant(np.random.randn(
input_dims[0], 12, 12, 6),
dtype=dtype,
name="c2")
q = math_ops.div(conv, c2, name="div")
edge = self.trt_incompatible_op(q, name="incompatible")
edge = math_ops.div(edge, edge, name="div1")
r = math_ops.add(edge, edge, name="add")
p = math_ops.sub(p, edge, name="sub")
q = math_ops.mul(q, edge, name="mul1")
s = math_ops.add(p, q, name="add1")
s = math_ops.sub(s, r, name="sub1")
array_ops.squeeze(s, name=self.output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
# TODO(aaroey): LayoutOptimizer adds additional nodes to the graph which
# breaks the connection check, fix it.
# - my_trt_op_0 should have ["mul", "sub", "div1", "mul1", "add1",
# "add", "sub1"];
# - my_trt_op_1 should have ["weights","conv", "div"]
expected_engines=["my_trt_op_0", "my_trt_op_1"],
expected_output_dims=(100, 12, 12, 6),
allclose_atol=1.e-03,
allclose_rtol=1.e-03)
def GraphFn(self, x):
conv_filter = constant_op.constant(np.random.randn(3, 3, 2, 1),
dtype=dtypes.float32)
x = nn.conv2d(input=x,
filter=conv_filter,
strides=[1, 1, 1, 1],
padding="SAME",
name="conv")
bias = constant_op.constant(np.random.randn(1, 10, 10, 1),
dtype=dtypes.float32)
x = math_ops.add(x, bias)
x = nn.relu(x)
return array_ops.identity(x, name="output")
def GetParams(self):
"""Test for multi connection neighboring nodes wiring tests in TF-TRT."""
dtype = dtypes.float32
input_name = "input"
input_dims = [2, 3, 7, 5]
output_name = "output"
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype,
shape=input_dims,
name=input_name)
e = constant_op.constant(np.random.normal(.05, .005, [3, 2, 3, 4]),
name="weights",
dtype=dtype)
conv = nn.conv2d(input=x,
filter=e,
data_format="NCHW",
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
b = constant_op.constant(np.random.normal(2.0, 1.0, [1, 4, 1, 1]),
name="bias",
dtype=dtype)
t = conv + b
b = constant_op.constant(np.random.normal(5.0, 1.0, [1, 4, 1, 1]),
name="bias",
dtype=dtype)
q = conv - b
edge = self.trt_incompatible_op(q)
b = constant_op.constant(np.random.normal(5.0, 1.0, [1, 4, 1, 1]),
name="bias",
dtype=dtype)
d = b + conv
edge3 = self.trt_incompatible_op(d)
edge1 = self.trt_incompatible_op(conv)
t = t - edge1
q = q + edge
t = t + q
t = t + d
t = t - edge3
array_ops.squeeze(t, name=output_name)
return trt_test.TfTrtIntegrationTestParams(gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[
(2, 4, 5, 4)
])
def GetParams(self):
"""Create a graph containing multiple segment."""
# TODO(aaroey): test graph with different dtypes.
dtype = dtypes.float32
input_name = "input"
input_dims = [100, 24, 24, 2]
output_name = "output"
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(dtype=dtype,
shape=[None] + input_dims[1:],
name=input_name)
with g.device("/GPU:0"):
conv_filter = constant_op.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]
],
name="weights",
dtype=dtype)
conv = nn.conv2d(input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
c1 = constant_op.constant(np.random.randn(
input_dims[0], 12, 12, 6),
dtype=dtype,
name="c1")
p = math_ops.mul(conv, c1, name="mul")
c2 = constant_op.constant(np.random.randn(
input_dims[0], 12, 12, 6),
dtype=dtype,
name="c2")
q = math_ops.div(conv, c2, name="div")
edge = self.trt_incompatible_op(q, name="incompatible")
edge = math_ops.div(edge, edge, name="div1")
r = math_ops.add(edge, edge, name="add")
p = math_ops.sub(p, edge, name="sub")
q = math_ops.mul(q, edge, name="mul1")
s = math_ops.add(p, q, name="add1")
s = math_ops.sub(s, r, name="sub1")
array_ops.squeeze(s, name=output_name)
return trt_test.TfTrtIntegrationTestParams(gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[
(100, 12, 12, 6)
])
def GraphFn(self, x):
conv_filter1 = constant_op.constant(np.ones([3, 3, 1, 8]),
name="weights1",
dtype=dtypes.float32)
bias1 = constant_op.constant(np.random.randn(8), dtype=dtypes.float32)
x = nn.conv2d(input=x,
filter=conv_filter1,
strides=[1, 1, 1, 1],
padding="SAME",
name="conv")
x = nn.bias_add(x, bias1)
x = nn.relu(x)
conv_filter2 = constant_op.constant(np.ones([3, 3, 8, 1]),
name="weights2",
dtype=dtypes.float32)
bias2 = constant_op.constant(np.random.randn(1), dtype=dtypes.float32)
x = nn.conv2d(input=x,
filter=conv_filter2,
strides=[1, 1, 1, 1],
padding="SAME",
name="conv")
x = nn.bias_add(x, bias2)
return array_ops.identity(x, name="output")
def GetParams(self):
"""Create a graph containing multiple segment."""
# TODO(aaroey): test graph with different dtypes.
dtype = dtypes.float32
input_name = "input"
input_dims = [100, 24, 24, 2]
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(
dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
with g.device("/GPU:0"):
conv_filter = constant_op.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
name="weights",
dtype=dtype)
conv = nn.conv2d(
input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
c1 = constant_op.constant(
np.random.randn(input_dims[0], 12, 12, 6), dtype=dtype, name="c1")
p = math_ops.mul(conv, c1, name="mul")
c2 = constant_op.constant(
np.random.randn(input_dims[0], 12, 12, 6), dtype=dtype, name="c2")
q = math_ops.div(conv, c2, name="div")
edge = self.trt_incompatible_op(q, name="incompatible")
edge = math_ops.div(edge, edge, name="div1")
r = math_ops.add(edge, edge, name="add")
p = math_ops.sub(p, edge, name="sub")
q = math_ops.mul(q, edge, name="mul1")
s = math_ops.add(p, q, name="add1")
s = math_ops.sub(s, r, name="sub1")
array_ops.squeeze(s, name=self.output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
# TODO(aaroey): LayoutOptimizer adds additional nodes to the graph which
# breaks the connection check, fix it.
# - my_trt_op_0 should have ["mul", "sub", "div1", "mul1", "add1",
# "add", "sub1"];
# - my_trt_op_1 should have ["weights","conv", "div"]
expected_engines=["my_trt_op_0", "my_trt_op_1"],
expected_output_dims=(100, 12, 12, 6),
allclose_atol=1.e-03,
allclose_rtol=1.e-03)
def GetParams(self):
"""Create a graph containing multiple segment."""
# TODO(aaroey): test graph with different dtypes.
dtype = dtypes.float32
input_name = "input"
input_dims = [100, 24, 24, 2]
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(dtype=dtype,
shape=[None] + input_dims[1:],
name=input_name)
with g.device("/GPU:0"):
conv_filter = constant_op.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]
],
name="weights",
dtype=dtype)
conv = nn.conv2d(input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
c1 = constant_op.constant(np.random.randn(
input_dims[0], 12, 12, 6),
dtype=dtype)
p = conv * c1
c2 = constant_op.constant(np.random.randn(
input_dims[0], 12, 12, 6),
dtype=dtype)
q = conv / c2
edge = self.trt_incompatible_op(q)
edge /= edge
r = edge + edge
p -= edge
q *= edge
s = p + q
s -= r
array_ops.squeeze(s, name=self.output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
num_expected_engines=2,
expected_output_dims=(100, 12, 12, 6),
allclose_atol=1.e-03,
allclose_rtol=1.e-03)
def conv2d(tensor_in,
n_filters,
filter_shape,
strides=None,
padding='SAME',
bias=True,
activation=None,
batch_norm=False):
"""Creates 2D convolutional subgraph with bank of filters.
Uses tf.nn.conv2d under the hood.
Creates a filter bank:
[filter_shape[0], filter_shape[1], tensor_in[3], n_filters]
and applies it to the input tensor.
Args:
tensor_in: input Tensor, 4D shape:
[batch, in_height, in_width, in_depth].
n_filters: number of filters in the bank.
filter_shape: Shape of filters, a list of ints, 1-D of length 2.
strides: A list of ints, 1-D of length 4. The stride of the sliding
window for each dimension of input.
padding: A string: 'SAME' or 'VALID'. The type of padding algorthim to use.
See the [comment here]
(https://www.tensorflow.org/api_docs/python/nn.html#convolution)
bias: Boolean, if to add bias.
activation: Activation Op, optional. If provided applied on the output.
batch_norm: Whether to apply batch normalization.
Returns:
A Tensor with resulting convolution.
"""
with vs.variable_scope('convolution'):
if strides is None:
strides = [1, 1, 1, 1]
input_shape = tensor_in.get_shape()
filter_shape = list(filter_shape) + [input_shape[3], n_filters]
filters = vs.get_variable('filters', filter_shape, dtypes.float32)
output = nn.conv2d(tensor_in, filters, strides, padding)
if bias:
bias_var = vs.get_variable('bias', [1, 1, 1, n_filters],
dtypes.float32)
output += bias_var
if batch_norm:
output = batch_normalize(output, convnet=True)
if activation:
output = activation(output)
return output
def GetParams(self):
"""Test for multi connection neighboring nodes wiring tests in TF-TRT."""
dtype = dtypes.float32
input_name = "input"
input_dims = [2, 3, 7, 5]
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
e = constant_op.constant(
np.random.normal(.05, .005, [3, 2, 3, 4]),
name="weights",
dtype=dtype)
conv = nn.conv2d(
input=x,
filter=e,
data_format="NCHW",
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
b = constant_op.constant(
np.random.normal(2.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
t = conv + b
b = constant_op.constant(
np.random.normal(5.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
q = conv - b
edge = math_ops.sigmoid(q)
b = constant_op.constant(
np.random.normal(5.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
d = b + conv
edge3 = math_ops.sigmoid(d)
edge1 = gen_math_ops.tan(conv)
t = t - edge1
q = q + edge
t = t + q
t = t + d
t = t - edge3
array_ops.squeeze(t, name=self.output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
expected_engines=["my_trt_op_0", "my_trt_op_1"],
expected_output_dims=(2, 4, 5, 4),
allclose_atol=1.e-03,
allclose_rtol=1.e-03)
def GetParams(self):
"""Test for multi connection neighboring nodes wiring tests in TF-TRT."""
dtype = dtypes.float32
input_name = "input"
input_dims = [2, 3, 7, 5]
output_name = "output"
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
e = constant_op.constant(
np.random.normal(.05, .005, [3, 2, 3, 4]),
name="weights",
dtype=dtype)
conv = nn.conv2d(
input=x,
filter=e,
data_format="NCHW",
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
b = constant_op.constant(
np.random.normal(2.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
t = conv + b
b = constant_op.constant(
np.random.normal(5.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
q = conv - b
edge = self.trt_incompatible_op(q)
b = constant_op.constant(
np.random.normal(5.0, 1.0, [1, 4, 1, 1]), name="bias", dtype=dtype)
d = b + conv
edge3 = self.trt_incompatible_op(d)
edge1 = self.trt_incompatible_op(conv)
t = t - edge1
q = q + edge
t = t + q
t = t + d
t = t - edge3
array_ops.squeeze(t, name=output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[[input_dims]],
output_names=[output_name],
expected_output_dims=[[[2, 4, 5, 4]]])
def conv2d(tensor_in,
n_filters,
filter_shape,
strides=None,
padding='SAME',
bias=True,
activation=None,
batch_norm=False):
"""Creates 2D convolutional subgraph with bank of filters.
Uses tf.nn.conv2d under the hood.
Creates a filter bank:
[filter_shape[0], filter_shape[1], tensor_in[3], n_filters]
and applies it to the input tensor.
Args:
tensor_in: input Tensor, 4D shape:
[batch, in_height, in_width, in_depth].
n_filters: number of filters in the bank.
filter_shape: Shape of filters, a list of ints, 1-D of length 2.
strides: A list of ints, 1-D of length 4. The stride of the sliding
window for each dimension of input.
padding: A string: 'SAME' or 'VALID'. The type of padding algorthim to use.
See the [comment here]
(https://www.tensorflow.org/api_docs/python/nn.html#convolution)
bias: Boolean, if to add bias.
activation: Activation Op, optional. If provided applied on the output.
batch_norm: Whether to apply batch normalization.
Returns:
A Tensor with resulting convolution.
"""
with vs.variable_scope('convolution'):
if strides is None:
strides = [1, 1, 1, 1]
input_shape = tensor_in.get_shape()
filter_shape = list(filter_shape) + [input_shape[3], n_filters]
filters = vs.get_variable('filters', filter_shape, dtypes.float32)
output = nn.conv2d(tensor_in, filters, strides, padding)
if bias:
bias_var = vs.get_variable('bias', [1, 1, 1, n_filters], dtypes.float32)
output += bias_var
if batch_norm:
output = batch_normalize(output, convnet=True)
if activation:
output = activation(output)
return output
def preact_conv2d(inputs,
num_outputs,
kernel_size,
stride=1,
padding='SAME',
activation_fn=nn.relu,
normalizer_fn=None,
normalizer_params=None,
weights_initializer=initializers.xavier_initializer(),
weights_regularizer=None,
reuse=None,
variables_collections=None,
outputs_collections=None,
trainable=True,
scope=None):
"""Adds a 2D convolution preceded by batch normalization and activation.
"""
with variable_scope.variable_scope(scope,
'Conv',
values=[inputs],
reuse=reuse) as sc:
inputs = ops.convert_to_tensor(inputs)
dtype = inputs.dtype.base_dtype
if normalizer_fn:
normalizer_params = normalizer_params or {}
inputs = normalizer_fn(inputs,
activation_fn=activation_fn,
**normalizer_params)
kernel_h, kernel_w = utils.two_element_tuple(kernel_size)
stride_h, stride_w = utils.two_element_tuple(stride)
num_filters_in = utils.last_dimension(inputs.get_shape(), min_rank=4)
weights_shape = [kernel_h, kernel_w, num_filters_in, num_outputs]
weights_collections = utils.get_variable_collections(
variables_collections, 'weights')
weights = variables.model_variable('weights',
shape=weights_shape,
dtype=dtype,
initializer=weights_initializer,
regularizer=weights_regularizer,
collections=weights_collections,
trainable=trainable)
outputs = nn.conv2d(inputs,
weights, [1, stride_h, stride_w, 1],
padding=padding)
return utils.collect_named_outputs(outputs_collections, sc.name,
outputs)
def _conv_and_pool_1(self, inp):
dtype = inp.dtype
conv_filter = constant_op.constant(
[[[[1., 0.5], [4., 6.], [0.5, 1.]]]], name="weights", dtype=dtype)
conv = nn.conv2d(input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
bias = constant_op.constant([4., 1.5], name="bias", dtype=dtype)
added = nn.bias_add(conv, bias, name="bias_add")
relu = nn.relu(added, "relu")
identity = array_ops.identity(relu, "identity")
pool = nn_ops.max_pool(identity, [1, 2, 2, 1], [1, 2, 2, 1],
"VALID",
name="max_pool")
return array_ops.squeeze(pool)
def GetParams(self):
"""Single vgg layer in NCHW unit tests in TF-TRT."""
dtype = dtypes.float32
input_name = "input"
input_dims = [5, 2, 8, 8]
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype,
shape=input_dims,
name=input_name)
x, _, _ = nn_impl.fused_batch_norm(
x,
np.random.randn(2).astype(np.float32),
np.random.randn(2).astype(np.float32),
mean=np.random.randn(2).astype(np.float32),
variance=np.random.randn(2).astype(np.float32),
data_format="NCHW",
is_training=False)
e = constant_op.constant(np.random.randn(1, 1, 2, 6),
name="weights",
dtype=dtype)
conv = nn.conv2d(input=x,
filter=e,
data_format="NCHW",
strides=[1, 1, 2, 2],
padding="SAME",
name="conv")
b = constant_op.constant(np.random.randn(6),
name="bias",
dtype=dtype)
t = nn.bias_add(conv, b, data_format="NCHW", name="biasAdd")
relu = nn.relu(t, "relu")
idty = array_ops.identity(relu, "ID")
v = nn_ops.max_pool(idty, [1, 1, 2, 2], [1, 1, 2, 2],
"VALID",
data_format="NCHW",
name="max_pool")
array_ops.squeeze(v, name="output")
return trt_test.TfTrtIntegrationTestParams(gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
num_expected_engines=1,
expected_output_dims=(5, 6,
2, 2),
allclose_atol=1.e-03,
allclose_rtol=1.e-03)
def GetParams(self):
"""Create a graph containing multiple segment."""
# TODO(aaroey): test graph with different dtypes.
dtype = dtypes.float32
input_name = "input"
input_dims = [100, 24, 24, 2]
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(
dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
with g.device("/GPU:0"):
conv_filter = constant_op.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
name="weights",
dtype=dtype)
conv = nn.conv2d(
input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
c1 = constant_op.constant(
np.random.randn(input_dims[0], 12, 12, 6), dtype=dtype)
p = conv * c1
c2 = constant_op.constant(
np.random.randn(input_dims[0], 12, 12, 6), dtype=dtype)
q = conv / c2
edge = self.trt_incompatible_op(q)
edge /= edge
r = edge + edge
p -= edge
q *= edge
s = p + q
s -= r
array_ops.squeeze(s, name=self.output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
num_expected_engines=2,
expected_output_dims=(100, 12, 12, 6),
allclose_atol=1.e-03,
allclose_rtol=1.e-03)
def GetParams(self):
"""Create a graph containing multiple segment."""
# TODO(aaroey): test graph with different dtypes.
dtype = dtypes.float32
input_name = "input"
input_dims = [100, 24, 24, 2]
output_name = "output"
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(
dtype=dtype, shape=input_dims, name=input_name)
with g.device("/GPU:0"):
conv_filter = constant_op.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
name="weights",
dtype=dtype)
conv = nn.conv2d(
input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
c1 = constant_op.constant(
np.random.randn(12, 12, 6), dtype=dtype, name="c1")
p = math_ops.mul(conv, c1, name="mul")
c2 = constant_op.constant(
np.random.randn(12, 12, 6), dtype=dtype, name="c2")
q = math_ops.div(conv, c2, name="div")
edge = self.trt_incompatible_op(q, name="incompatible")
edge = math_ops.div(edge, edge, name="div1")
r = math_ops.add(edge, edge, name="add")
p = math_ops.sub(p, edge, name="sub")
q = math_ops.mul(q, edge, name="mul1")
s = math_ops.add(p, q, name="add1")
s = math_ops.sub(s, r, name="sub1")
array_ops.squeeze(s, name=output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[(100, 12, 12, 6)])
def GetParams(self):
"""Single vgg layer in NCHW unit tests in TF-TRT."""
dtype = dtypes.float32
input_name = "input"
input_dims = [5, 2, 8, 8]
g = ops.Graph()
with g.as_default():
x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name)
x, _, _ = nn_impl.fused_batch_norm(
x,
np.random.randn(2).astype(np.float32),
np.random.randn(2).astype(np.float32),
mean=np.random.randn(2).astype(np.float32),
variance=np.random.randn(2).astype(np.float32),
data_format="NCHW",
is_training=False)
e = constant_op.constant(
np.random.randn(1, 1, 2, 6), name="weights", dtype=dtype)
conv = nn.conv2d(
input=x,
filter=e,
data_format="NCHW",
strides=[1, 1, 2, 2],
padding="SAME",
name="conv")
b = constant_op.constant(np.random.randn(6), name="bias", dtype=dtype)
t = nn.bias_add(conv, b, data_format="NCHW", name="biasAdd")
relu = nn.relu(t, "relu")
idty = array_ops.identity(relu, "ID")
v = nn_ops.max_pool(
idty, [1, 1, 2, 2], [1, 1, 2, 2],
"VALID",
data_format="NCHW",
name="max_pool")
array_ops.squeeze(v, name="output")
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
num_expected_engines=1,
expected_output_dims=(5, 6, 2, 2),
allclose_atol=1.e-03,
allclose_rtol=1.e-03)
def GetParams(self):
"""Create a graph containing single segment."""
# TODO(aaroey): test graph with different dtypes.
dtype = dtypes.float32
input_name = "input"
input_dims = [100, 24, 24, 2]
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(
dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
with g.device("/GPU:0"):
conv_filter = constant_op.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
name="weights",
dtype=dtype)
conv = nn.conv2d(
input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
bias = constant_op.constant(
[4., 1.5, 2., 3., 5., 7.], name="bias", dtype=dtype)
added = nn.bias_add(conv, bias, name="bias_add")
relu = nn.relu(added, "relu")
identity = array_ops.identity(relu, "identity")
pool = nn_ops.max_pool(
identity, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool")
array_ops.squeeze(pool, name=self.output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
# TODO(aaroey): LayoutOptimizer adds additional nodes to the graph which
# breaks the connection check, fix it.
# - my_trt_op_0 should have ["weights", "conv", "bias", "bias_add",
# "relu", "identity", "max_pool"]
expected_engines=["my_trt_op_0"],
expected_output_dims=(100, 6, 6, 6),
allclose_atol=1.e-03,
allclose_rtol=1.e-03)
def _GetMetaGraph(self, batch_size=14, image_dim=12, optimizer_scope_name=''):
"""A simple layered graph with conv, an intermediate op, and a ReLU."""
graph = ops.Graph()
with graph.as_default():
random_seed.set_random_seed(1)
current_activation = variable_scope.get_variable(
name='start', shape=[batch_size, image_dim, image_dim, 5])
conv_filter = variable_scope.get_variable(
name='filter', shape=[5, 5, 5, 5])
for layer_number in range(10):
with variable_scope.variable_scope('layer_{}'.format(layer_number)):
after_conv = nn.conv2d(current_activation, conv_filter, [1, 1, 1, 1],
'SAME')
current_activation = 2. * after_conv
current_activation = nn.relu(current_activation)
loss = math_ops.reduce_mean(current_activation)
with ops.name_scope(optimizer_scope_name):
optimizer = train.AdamOptimizer(0.001)
train_op = optimizer.minimize(loss)
init_op = variables.global_variables_initializer()
metagraph = train.export_meta_graph()
return (metagraph, init_op.name, train_op.name, loss.name)
def get_multi_engine_graph_def(mode="FP32"):
"""Create a simple graph and return its graph_def."""
dtype = dtypes.float32
if mode.upper() == "FP16":
dtype = dtypes.float16
else:
pass
g = ops.Graph()
with g.as_default():
x = aops.placeholder(shape=[None, 3, 7, 5], name="input", dtype=dtype)
with g.name_scope("Global_scope"):
with g.name_scope("first_scope"):
e = cop.constant(
np.random.randn(3, 2, 3, 4), name="weights", dtype=dtype)
conv = nn.conv2d(
input=x,
filter=e,
data_format="NCHW",
strides=[1, 1, 1, 1],
padding="VALID",
name="conv")
b = cop.constant(np.random.randn(1, 4, 1, 1), name="bias1", dtype=dtype)
t = conv * b
b = cop.constant(np.random.randn(1, 4, 1, 1), name="bias2", dtype=dtype)
q = conv / b
edge = mops.sin(q)
edge1 = mops.cos(conv)
with g.name_scope("test_scope"):
de = edge + edge1
t -= edge1
q *= edge
t += q
t -= de
k = aops.squeeze(t, name="output")
print(k.dtype)
return g.as_graph_def()
def GetParams(self):
"""Create a graph containing single segment."""
# TODO(aaroey): test graph with different dtypes.
dtype = dtypes.float32
input_name = "input"
input_dims = [100, 24, 24, 2]
output_name = "output"
g = ops.Graph()
with g.as_default():
inp = array_ops.placeholder(
dtype=dtype, shape=[None] + input_dims[1:], name=input_name)
with g.device("/GPU:0"):
conv_filter = constant_op.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
name="weights",
dtype=dtype)
conv = nn.conv2d(
input=inp,
filter=conv_filter,
strides=[1, 2, 2, 1],
padding="SAME",
name="conv")
bias = constant_op.constant([4., 1.5, 2., 3., 5., 7.],
name="bias",
dtype=dtype)
added = nn.bias_add(conv, bias, name="bias_add")
relu = nn.relu(added, "relu")
identity = array_ops.identity(relu, "identity")
pool = nn_ops.max_pool(
identity, [1, 2, 2, 1], [1, 2, 2, 1], "VALID", name="max_pool")
array_ops.squeeze(pool, name=output_name)
return trt_test.TfTrtIntegrationTestParams(
gdef=g.as_graph_def(),
input_names=[input_name],
input_dims=[input_dims],
output_names=[output_name],
expected_output_dims=[(100, 6, 6, 6)])
def _RunGraphWithConfig(self, config, batch_size=14, image_dim=12):
"""Run a simple layered graph with conv, an intermediate op, and a ReLU."""
graph = ops.Graph()
with graph.as_default():
random_seed.set_random_seed(1)
current_activation = variable_scope.get_variable(
name='start', shape=[batch_size, image_dim, image_dim, 5])
conv_filter = variable_scope.get_variable(
name='filter', shape=[5, 5, 5, 5])
for layer_number in range(10):
with variable_scope.variable_scope('layer_{}'.format(layer_number)):
after_conv = nn.conv2d(current_activation, conv_filter, [1, 1, 1, 1],
'SAME')
current_activation = 2. * after_conv
current_activation = nn.relu(current_activation)
loss = math_ops.reduce_mean(current_activation)
optimizer = train.AdamOptimizer(0.001)
train_op = optimizer.minimize(loss)
init_op = variables.global_variables_initializer()
with session.Session(config=config, graph=graph) as sess:
sess.run(init_op)
sess.run(train_op)
sess.run(train_op)
return sess.run(loss)
def testSinglePartitionedVariable(self):
"""Ensures partitioned variables fail cleanly with freeze graph."""
checkpoint_prefix = os.path.join(self.get_temp_dir(), "saved_checkpoint")
checkpoint_state_name = "checkpoint_state"
input_graph_name = "input_graph.pb"
output_graph_name = "output_graph.pb"
# Create a graph with partition variables. When weights are partitioned into
# a single partition, the weights variable is followed by a identity ->
# identity (an additional identity node).
partitioner = partitioned_variables.fixed_size_partitioner(1)
with ops.Graph().as_default():
with variable_scope.variable_scope("part", partitioner=partitioner):
batch_size, height, width, depth = 5, 128, 128, 3
input1 = array_ops.zeros(
(batch_size, height, width, depth), name="input1")
input2 = array_ops.zeros(
(batch_size, height, width, depth), name="input2")
num_nodes = depth
filter1 = variable_scope.get_variable("filter", [num_nodes, num_nodes])
filter2 = array_ops.reshape(filter1, [1, 1, num_nodes, num_nodes])
conv = nn.conv2d(
input=input1, filter=filter2, strides=[1, 1, 1, 1], padding="SAME")
node = math_ops.add(conv, input2, name="test/add")
node = nn.relu6(node, name="test/relu6")
# Save graph and checkpoints.
sess = session.Session()
sess.run(variables.global_variables_initializer())
saver = saver_lib.Saver()
checkpoint_path = saver.save(
sess,
checkpoint_prefix,
global_step=0,
latest_filename=checkpoint_state_name)
graph_io.write_graph(sess.graph, self.get_temp_dir(), input_graph_name)
# Ensure this graph has partition variables.
self.assertTrue([
tensor.name.split(":")[0]
for op in sess.graph.get_operations()
for tensor in op.values()
if re.search(r"/part_\d+/", tensor.name)
])
# Test freezing graph doesn't make it crash.
output_node_names = "save/restore_all"
output_graph_path = os.path.join(self.get_temp_dir(), output_graph_name)
return_value = freeze_graph.freeze_graph_with_def_protos(
input_graph_def=sess.graph_def,
input_saver_def=None,
input_checkpoint=checkpoint_path,
output_node_names=output_node_names,
restore_op_name="save/restore_all", # default value
filename_tensor_name="save/Const:0", # default value
output_graph=output_graph_path,
clear_devices=False,
initializer_nodes="")
self.assertTrue(return_value, -1)
def convolution2d(x,
num_output_channels,
kernel_size,
activation_fn=None,
stride=(1, 1),
padding='SAME',
weight_init=None,
bias_init=standard_ops.constant_initializer(0.),
num_input_channels=None,
name=None,
weight_collections=None,
bias_collections=None,
weight_regularizer=None,
create_summaries=True):
"""Adds the parameters for a conv2d layer and returns the output.
A neural network convolution layer is generally defined as:
\\\\(y = f(conv2d(w, x) + b)\\\\) where **f** is given by `activation_fn`,
**conv2d** is `nn.conv2d` and `x` has shape
`[batch, height, width, channels]`
This op creates `w` and optionally `b` and adds various summaries that can be
useful for visualizing learning or diagnosing training problems. Bias can be
disabled by setting `bias_init` to `None`.
The variable creation is compatible with `tf.variable_scope` and so can be
reused with `tf.variable_scope` or `tf.make_template`.
In almost all cases, the input channels can be inferred from the shape
of `x`, but if it is unspecified or additional size checks are
desired, then `num_input_channels` can be specified.
Most of the details of variable creation can be controlled by specifying the
initializers (`weight_init` and `bias_init`) and which collections to place
the created variables in (`weight_collections` and `bias_collections`).
A per layer regularization can be specified by setting `weight_regularizer`.
This is only applied to weights and not the bias.
Args:
x: The input `Tensor`.
num_output_channels: The number of output channels (i.e. the size of
dim[3]).
kernel_size: A length 2 `list` or `tuple` containing the kernel size.
activation_fn: A function that requires a single Tensor that is applied as a
non-linearity.
stride: A length 2 `list` or `tuple` specifying the stride of the sliding
window across the image.
padding: A `string` from: "SAME", "VALID". The type of padding algorithm to
use.
weight_init: An optional initialization. If not specified, uses Xavier
initialization (see `tf.learn.xavier_initializer`).
bias_init: An initializer for the bias, defaults to 0. Set to`None` in order
to disable bias.
num_input_channels: The length of the channel dimension in the input.
name: The name for this operation is used to name operations and to find
variables. If specified it must be unique for this scope, otherwise a
unique name starting with "convolution2d" will be created. See
`tf.variable_op_scope` for details.
weight_collections: List of graph collections for just weights.
bias_collections: List of graph collections for just bias.
weight_regularizer: A regularizer like the result of
`tf.learn.l1_regularizer` or `tf.learn.l2_regularizer`.
create_summaries: Set to false to disable summaries.
Returns:
The result of applying a fully connected layer.
Raises:
ValueError: if `x` is not rank 4; or `x`'s channel dimension is not known
and `num_input_channels` is not specified.
"""
with variable_scope.variable_op_scope([x], name, 'convolution2d') as vs:
# Check rank and if num_input_channels is specified, make sure it matches.
x.get_shape().assert_is_compatible_with([None, None, None,
num_input_channels])
if not num_input_channels:
if x.get_shape().dims is None or x.get_shape().dims[3].value is None:
raise ValueError(
'If x has an unknown channels dimension then num_input_channels '
'must be specified; shape: %s num_input_channels: %s'
% (x.get_shape(), num_input_channels))
else:
num_input_channels = x.get_shape().dims[3].value
# QQQ: Should we accept a scalar for a square convolution?
if len(kernel_size) != 2:
raise ValueError('kernel_size must be length 2: ' % kernel_size)
if len(stride) != 2:
raise ValueError('stride must be length 2: ' % kernel_size)
stride = [1, stride[0], stride[1], 1]
shape = [kernel_size[0], kernel_size[1], num_input_channels,
num_output_channels]
patch_size = kernel_size[0] * kernel_size[1]
weight_init = weight_init or xavier_initializer(
num_input_channels * patch_size, num_output_channels * patch_size)
dtype = x.dtype.base_dtype
w = variable_scope.get_variable('weights',
shape=shape,
dtype=dtype,
initializer=weight_init,
collections=weight_collections)
if not vs.reuse and create_summaries:
_add_histogram_summary(w)
y = nn.conv2d(x, w, stride, padding)
# Regularization is only applied to the weights and not bias.
if weight_regularizer:
_apply_regularization(w, weight_regularizer)
if bias_init:
b = _bias_variable(
num_output_channels, dtype, bias_init, bias_collections,
create_summaries)
y = nn.bias_add(y, b)
if create_summaries:
return _apply_activation_with_summaries(y, activation_fn)
if activation_fn:
y = activation_fn(y)
return y
def conv2d(x, w): """conv2d returns a 2d convolution layer with full stride.""" return nn.conv2d(x, w, strides=[1, 1, 1, 1], padding='SAME')
def convolution2d(inputs,
num_outputs,
kernel_size,
stride=1,
padding='SAME',
activation_fn=nn.relu,
normalizer_fn=None,
normalizer_params=None,
weights_initializer=initializers.xavier_initializer(),
weights_regularizer=None,
biases_initializer=init_ops.zeros_initializer,
biases_regularizer=None,
reuse=None,
variables_collections=None,
outputs_collections=None,
trainable=True,
scope=None):
"""Adds a 2D convolution followed by an optional batch_norm layer.
`convolution2d` creates a variable called `weights`, representing the
convolutional kernel, that is convolved with the `inputs` to produce a
`Tensor` of activations. If a `normalizer_fn` is provided (such as
`batch_norm`), it is then applied. Otherwise, if `normalizer_fn` is
None and a `biases_initializer` is provided then a `biases` variable would be
created and added the activations. Finally, if `activation_fn` is not `None`,
it is applied to the activations as well.
Args:
inputs: a 4-D tensor `[batch_size, height, width, channels]`.
num_outputs: integer, the number of output filters.
kernel_size: a list of length 2 `[kernel_height, kernel_width]` of
of the filters. Can be an int if both values are the same.
stride: a list of length 2 `[stride_height, stride_width]`.
Can be an int if both strides are the same. Note that presently
both strides must have the same value.
padding: one of `VALID` or `SAME`.
activation_fn: activation function.
normalizer_fn: normalization function to use instead of `biases`. If
`normalize_fn` is provided then `biases_initializer` and
`biases_regularizer` are ignored and `biases` are not created nor added.
normalizer_params: normalization function parameters.
weights_initializer: An initializer for the weights.
weights_regularizer: Optional regularizer for the weights.
biases_initializer: An initializer for the biases. If None skip biases.
biases_regularizer: Optional regularizer for the biases.
reuse: whether or not the layer and its variables should be reused. To be
able to reuse the layer scope must be given.
variables_collections: optional list of collections for all the variables or
a dictionay containing a different list of collection per variable.
outputs_collections: collection to add the outputs.
trainable: If `True` also add variables to the graph collection
`GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable).
scope: Optional scope for `variable_op_scope`.
Returns:
a tensor representing the output of the operation.
"""
with variable_scope.variable_op_scope([inputs],
scope, 'Conv', reuse=reuse) as sc:
dtype = inputs.dtype.base_dtype
kernel_h, kernel_w = utils.two_element_tuple(kernel_size)
stride_h, stride_w = utils.two_element_tuple(stride)
num_filters_in = utils.last_dimension(inputs.get_shape(), min_rank=4)
weights_shape = [kernel_h, kernel_w,
num_filters_in, num_outputs]
weights_collections = utils.get_variable_collections(
variables_collections, 'weights')
weights = variables.model_variable('weights',
shape=weights_shape,
dtype=dtype,
initializer=weights_initializer,
regularizer=weights_regularizer,
collections=weights_collections,
trainable=trainable)
outputs = nn.conv2d(inputs, weights, [1, stride_h, stride_w, 1],
padding=padding)
if normalizer_fn:
normalizer_params = normalizer_params or {}
outputs = normalizer_fn(outputs, **normalizer_params)
else:
if biases_initializer is not None:
biases_collections = utils.get_variable_collections(
variables_collections, 'biases')
biases = variables.model_variable('biases',
shape=[num_outputs,],
dtype=dtype,
initializer=biases_initializer,
regularizer=biases_regularizer,
collections=biases_collections,
trainable=trainable)
outputs = nn.bias_add(outputs, biases)
if activation_fn:
outputs = activation_fn(outputs)
return utils.collect_named_outputs(outputs_collections, sc.name, outputs)
def conv2d_leaders(inputs,
num_outputs,
kernel_size,
rates=[1],
stride=1,
padding='SAME',
activation_fn=nn.relu,
normalizer_fn=None,
normalizer_params=None,
weights_initializer=initializers.xavier_initializer(),
weights_regularizer=None,
biases_initializer=init_ops.zeros_initializer,
biases_regularizer=None,
reuse=None,
variables_collections=None,
outputs_collections=None,
trainable=True,
scope=None,):
"""Adds a 2D convolution followed by an optional batch_norm layer.
`convolution2d` creates a variable called `weights`, representing the
convolutional kernel, that is convolved with the `inputs` to produce a
`Tensor` of activations. If a `normalizer_fn` is provided (such as
`batch_norm`), it is then applied. Otherwise, if `normalizer_fn` is
None and a `biases_initializer` is provided then a `biases` variable would be
created and added the activations. Finally, if `activation_fn` is not `None`,
it is applied to the activations as well.
Performs a'trous convolution with input stride equal to rate if rate is
greater than one.
Args:
inputs: a 4-D tensor `[batch_size, height, width, channels]`.
num_outputs: integer, the number of output filters.
kernel_size: a list of length 2 `[kernel_height, kernel_width]` of
of the filters. Can be an int if both values are the same.
stride: a list of length 2 `[stride_height, stride_width]`.
Can be an int if both strides are the same. Note that presently
both strides must have the same value.
padding: one of `VALID` or `SAME`.
rate: integer. If less than or equal to 1, a standard convolution is used.
If greater than 1, than the a'trous convolution is applied and `stride`
must be set to 1.
activation_fn: activation function.
normalizer_fn: normalization function to use instead of `biases`. If
`normalize_fn` is provided then `biases_initializer` and
`biases_regularizer` are ignored and `biases` are not created nor added.
normalizer_params: normalization function parameters.
weights_initializer: An initializer for the weights.
weights_regularizer: Optional regularizer for the weights.
biases_initializer: An initializer for the biases. If None skip biases.
biases_regularizer: Optional regularizer for the biases.
reuse: whether or not the layer and its variables should be reused. To be
able to reuse the layer scope must be given.
variables_collections: optional list of collections for all the variables or
a dictionay containing a different list of collection per variable.
outputs_collections: collection to add the outputs.
trainable: If `True` also add variables to the graph collection
`GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable).
scope: Optional scope for `variable_op_scope`.
Returns:
a tensor representing the output of the operation.
Raises:
ValueError: if both 'rate' and `stride` are larger than one.
"""
with variable_scope.variable_scope(scope, 'Conv', [inputs],
reuse=reuse) as sc:
inputs = ops.convert_to_tensor(inputs)
dtype = inputs.dtype.base_dtype
# inshape = tf.shape(inputs)
# Leading kernel size.
kernel_h, kernel_w = utils.two_element_tuple(kernel_size)
stride_h, stride_w = utils.two_element_tuple(stride)
num_filters_in = utils.last_dimension(inputs.get_shape(), min_rank=4)
# Weights variable.
weights_shape = [kernel_h, kernel_w,
num_filters_in, num_outputs]
weights_collections = utils.get_variable_collections(
variables_collections, 'weights')
weights = variables.model_variable('weights',
shape=weights_shape,
dtype=dtype,
initializer=weights_initializer,
regularizer=weights_regularizer,
collections=weights_collections,
trainable=trainable)
# # Bias variable.
# biases = None
# if biases_initializer is not None:
# biases_collections = utils.get_variable_collections(
# variables_collections, 'biases')
# biases = variables.model_variable('biases',
# shape=[num_outputs, ],
# dtype=dtype,
# initializer=biases_initializer,
# regularizer=biases_regularizer,
# collections=biases_collections,
# trainable=trainable)
# Convolution at different scales.
outputs_pool = []
for rate in rates:
if rate > 1:
conv = nn.atrous_conv2d(inputs, weights, rate, padding='SAME')
else:
conv = nn.conv2d(inputs, weights, [1, 1, 1, 1], padding='SAME')
outputs_pool.append(conv)
# 'Pooling' at different scales. A bit hacky. Use of concat + max_pool?
outputs = None
outputs_pool.reverse()
for node in outputs_pool:
if outputs is None:
outputs = node
else:
outputs = tf.maximum(outputs, node)
# # Add bias?
# if biases is not None:
# outputs = tf.nn.bias_add(outputs, biases)
# Fix padding and stride. A bit hacky too and not so efficient!
if padding == 'VALID' or stride > 1:
padfilter = np.zeros(shape=(kernel_h, kernel_w, num_filters_in, 1),
dtype=dtype)
x = (kernel_h - 1) / 2
y = (kernel_w - 1) / 2
padfilter[x, y, :, 0] = 1.
outputs = tf.nn.depthwise_conv2d(outputs, padfilter,
[1, stride_h, stride_w, 1],
padding=padding)
# Batch norm / bias and activation...
if normalizer_fn is not None:
normalizer_params = normalizer_params or {}
outputs = normalizer_fn(outputs, **normalizer_params)
else:
if biases_initializer is not None:
biases_collections = utils.get_variable_collections(
variables_collections, 'biases')
biases = variables.model_variable('biases',
shape=[num_outputs, ],
dtype=dtype,
initializer=biases_initializer,
regularizer=biases_regularizer,
collections=biases_collections,
trainable=trainable)
outputs = nn.bias_add(outputs, biases)
if activation_fn is not None:
outputs = activation_fn(outputs)
return utils.collect_named_outputs(outputs_collections,
sc.name, outputs)
def legacy_convolution2d(x,
num_output_channels,
kernel_size,
activation_fn=None,
stride=(1, 1),
padding='SAME',
weight_init=initializers.xavier_initializer_conv2d(),
bias_init=standard_ops.zeros_initializer,
name=None,
weight_collections=(ops.GraphKeys.WEIGHTS,),
bias_collections=(ops.GraphKeys.BIASES,),
output_collections=(ops.GraphKeys.ACTIVATIONS,),
trainable=True,
weight_regularizer=None,
bias_regularizer=None):
# pylint: disable=g-docstring-has-escape
"""Adds the parameters for a conv2d layer and returns the output.
A neural network convolution layer is generally defined as:
\\\\(y = f(conv2d(w, x) + b)\\\\) where **f** is given by `activation_fn`,
**conv2d** is `tf.nn.conv2d` and `x` has shape
`[batch, height, width, channels]`. The output of this op is of shape
`[batch, out_height, out_width, num_output_channels]`, where `out_width` and
`out_height` are determined by the `padding` argument. See `conv2D` for
details.
This op creates `w` and optionally `b` and adds various summaries that can be
useful for visualizing learning or diagnosing training problems. Bias can be
disabled by setting `bias_init` to `None`.
The variable creation is compatible with `tf.variable_scope` and so can be
reused with `tf.variable_scope` or `tf.make_template`.
Most of the details of variable creation can be controlled by specifying the
initializers (`weight_init` and `bias_init`) and which collections to place
the created variables in (`weight_collections` and `bias_collections`).
A per layer regularization can be specified by setting `weight_regularizer`.
This is only applied to weights and not the bias.
Args:
x: A 4-D input `Tensor`.
num_output_channels: The number of output channels (i.e. the size of the
last dimension of the output).
kernel_size: A length 2 `list` or `tuple` containing the kernel size.
activation_fn: A function that requires a single Tensor that is applied as a
non-linearity.
stride: A length 2 `list` or `tuple` specifying the stride of the sliding
window across the image.
padding: A `string` from: "SAME", "VALID". The type of padding algorithm to
use.
weight_init: An optional initialization. If not specified, uses Xavier
initialization (see `tf.learn.xavier_initializer`).
bias_init: An initializer for the bias, defaults to 0. Set to`None` in order
to disable bias.
name: The name for this operation is used to name operations and to find
variables. If specified it must be unique for this scope, otherwise a
unique name starting with "convolution2d" will be created. See
`tf.variable_op_scope` for details.
weight_collections: List of graph collections to which weights are added.
bias_collections: List of graph collections to which biases are added.
output_collections: List of graph collections to which outputs are added.
trainable: If `True` also add variables to the graph collection
`GraphKeys.TRAINABLE_VARIABLES` (see tf.Variable).
weight_regularizer: A regularizer like the result of
`l1_regularizer` or `l2_regularizer`. Used for weights.
bias_regularizer: A regularizer like the result of
`l1_regularizer` or `l2_regularizer`. Used for biases.
Returns:
The result of applying a 2-D convolutional layer.
Raises:
ValueError: If `kernel_size` or `stride` are not length 2.
"""
with variable_scope.variable_op_scope([x], name, 'convolution2d'):
num_input_channels = x.get_shape().dims[3].value
if len(kernel_size) != 2:
raise ValueError('kernel_size must be length 2: %d ' % kernel_size)
if len(stride) != 2:
raise ValueError('stride must be length 2: %d' % stride)
stride = [1, stride[0], stride[1], 1]
shape = [kernel_size[0], kernel_size[1], num_input_channels,
num_output_channels]
dtype = x.dtype.base_dtype
weight_collections = set(list(weight_collections or []) +
[ops.GraphKeys.VARIABLES])
w = variable_scope.get_variable('weights',
shape=shape,
dtype=dtype,
initializer=weight_init,
collections=weight_collections,
regularizer=weight_regularizer,
trainable=trainable)
y = nn.conv2d(x, w, stride, padding)
if bias_init is not None:
bias_collections = set(list(bias_collections or []) +
[ops.GraphKeys.VARIABLES])
b = variable_scope.get_variable('bias',
shape=[num_output_channels],
dtype=dtype,
initializer=bias_init,
collections=bias_collections,
regularizer=bias_regularizer,
trainable=trainable)
y = nn.bias_add(y, b)
return _apply_activation(y, activation_fn, output_collections)
def loop_fn(i):
x1 = array_ops.gather(x, i)
return nn.conv2d(
x1, filt, strides=[1, 2, 2, 1], padding="VALID", data_format="NHWC")