def build_test_residual_graph(self, backend):
"""Create a residual model.
The graph contains a residual connection, where the output of conv0 and
conv2 is added at the end."""
np_dtype = test_backend_dtypes[backend]
self.expected_dtype = datatypes.np_to_smaug_type[np_dtype]
with Graph(name="test_residual_graph", backend=backend) as graph:
input_tensor = Tensor(
data_layout=types_pb2.NCHW,
tensor_data=np.random.rand(1, 1, 28, 28).astype(np_dtype))
filter_tensor0 = Tensor(
data_layout=types_pb2.NCHW,
tensor_data=np.random.rand(64, 1, 3, 3).astype(np_dtype))
filter_tensor1 = Tensor(
data_layout=types_pb2.NCHW,
tensor_data=np.random.rand(64, 1, 3, 3).astype(np_dtype))
filter_tensor2 = Tensor(
data_layout=types_pb2.NCHW,
tensor_data=np.random.rand(64, 64, 3, 3).astype(np_dtype))
bn_mean_tensor = Tensor(
data_layout=types_pb2.NC,
tensor_data=np.random.rand(1, 64).astype(np_dtype))
bn_var_tensor = Tensor(
data_layout=types_pb2.NC,
tensor_data=np.random.rand(1, 64).astype(np_dtype))
bn_gamma_tensor = Tensor(
data_layout=types_pb2.NC,
tensor_data=np.random.rand(1, 64).astype(np_dtype))
bn_beta_tensor = Tensor(
data_layout=types_pb2.NC,
tensor_data=np.random.rand(1, 64).astype(np_dtype))
act = data_op.input_data(input_tensor, "input")
x = nn_ops.convolution(
act, filter_tensor0, stride=[1, 1], padding="same", name="conv0")
out = nn_ops.convolution(
act, filter_tensor1, stride=[1, 1], padding="same", name="conv1")
out = nn_ops.batch_norm(
out, bn_mean_tensor, bn_var_tensor, bn_gamma_tensor, bn_beta_tensor,
name="bn")
out = activation_ops.relu(out, "relu")
out = nn_ops.convolution(
out, filter_tensor2, stride=[1, 1], padding="same", name="conv2")
out = math_ops.add(x, out, "add")
out = math_ops.mul(x, out, "mul")
# Concatenate the channel dimension of x and out.
axis = 1 if out.shape.layout == types_pb2.NCHW else 3
out = array_ops.concat([x, out], axis, "concat")
# Evenly split the tensor into 4 over the channel dimension.
out0, out1, out2, out3 = array_ops.split(out, 4, axis, "split")
out = math_ops.mul(
math_ops.add(out0, out1, "add1"), math_ops.add(out2, out3, "add2"),
"mul1")
self.test_graph, _ = graph.to_proto()
self.backend = backend
self.alignment = global_vars.backend_alignment[
self.test_graph.backend]
def build_test_sequential_graph(self, backend):
"""Create a sequential model."""
np_dtype = test_backend_dtypes[backend]
self.expected_dtype = datatypes.np_to_smaug_type[np_dtype]
with Graph(name="test_sequential_graph", backend=backend) as graph:
input_tensor = Tensor(
data_layout=types_pb2.NCHW,
tensor_data=np.random.rand(1, 3, 28, 28).astype(np_dtype))
filter_tensor0 = Tensor(
data_layout=types_pb2.NCHW,
tensor_data=np.random.rand(64, 3, 3, 3).astype(np_dtype))
filter_tensor1 = Tensor(
data_layout=types_pb2.NCHW,
tensor_data=np.random.rand(64, 64, 3, 3).astype(np_dtype))
weight_tensor0 = Tensor(
data_layout=types_pb2.NC,
tensor_data=np.random.rand(254, 12544).astype(np_dtype))
weight_tensor1 = Tensor(
data_layout=types_pb2.NC,
tensor_data=np.random.rand(10, 254).astype(np_dtype))
bn_mean_tensor = Tensor(
data_layout=types_pb2.NC,
tensor_data=np.random.rand(1, 64).astype(np_dtype))
bn_var_tensor = Tensor(
data_layout=types_pb2.NC,
tensor_data=np.random.rand(1, 64).astype(np_dtype))
bn_gamma_tensor = Tensor(
data_layout=types_pb2.NC,
tensor_data=np.random.rand(1, 64).astype(np_dtype))
bn_beta_tensor = Tensor(
data_layout=types_pb2.NC,
tensor_data=np.random.rand(1, 64).astype(np_dtype))
out = data_op.input_data(input_tensor, "input")
out = nn_ops.convolution(
out, filter_tensor0, stride=[1, 1], padding="same", name="conv0")
out = activation_ops.relu(out, "conv0_relu")
out = nn_ops.batch_norm(
out, bn_mean_tensor, bn_var_tensor, bn_gamma_tensor, bn_beta_tensor,
name="bn")
out = nn_ops.convolution(
out, filter_tensor1, stride=[1, 1], padding="same", name="conv1")
out = activation_ops.relu(out, "conv1_relu")
out = nn_ops.max_pool(out, pool_size=[2, 2], stride=[2, 2], name="pool")
out = array_ops.flatten(out, "flatten")
out = nn_ops.mat_mul(out, weight_tensor0, name="fc0")
out = activation_ops.relu(out, "fc0_relu")
out = nn_ops.mat_mul(out, weight_tensor1, name="fc1")
out = array_ops.expand_dims(out, 1, "expand_dims")
out = array_ops.squeeze(out, 1, "squeeze")
out = array_ops.reshape(out, [2, 5], types_pb2.NC, "reshape")
out = array_ops.repeat(out, [4, 2], "repeat")
out = array_ops.stack(out, 4, 1, "stack")
out0, out1, out2, out3 = array_ops.unstack(out, 1, "unstack")
out0 = array_ops.reshape(out0, [1, 1, 8, 10], types_pb2.NCHW, "reshape")
out0 = array_ops.padding(out0, [0, 0, 0, 0, 1, 1, 1, 1], "padding")
self.test_graph, _ = graph.to_proto()
self.backend = backend
self.alignment = global_vars.backend_alignment[backend]
def test_convolution(self):
batch = 4
width = 8
height = 8
channels = 32
filter_height = 3
filter_width = 3
num_filters = 128
tf_inputs = tf.Variable(
initializer(shape=[batch, height, width, channels], dtype=self.dtype))
tf_filters = tf.Variable(
initializer(
shape=[filter_height, filter_width, channels, num_filters],
dtype=self.dtype))
tf_results = tf.nn.conv2d(
tf_inputs, tf_filters, strides=[1, 1], padding="SAME",
data_format="NHWC", dilations=None)
inputs = Tensor(data_layout=types_pb2.NHWC, tensor_data=tf_inputs.numpy())
filters = Tensor(
data_layout=types_pb2.NHWC,
tensor_data=np.transpose(tf_filters.numpy(), (3, 0, 1, 2)))
with Graph(name=self.graph_name, backend=self.backend) as graph:
nn_ops.convolution(inputs, filters, stride=[1, 1], padding="same")
self.runAndValidate(graph, tf_results, decimal=2)
def test_fusing_activation_functions(self):
"""Test activation function when they are fused with other operators."""
with Graph("test_graph", "SMV") as test_graph:
input_tensor = Tensor(data_layout=types_pb2.NHWC,
tensor_data=np.random.rand(
*self.tensor_shape).astype(np.float16))
filter_tensor = Tensor(data_layout=types_pb2.NHWC,
tensor_data=np.random.rand(
32, 3, 3, 32).astype(np.float16))
weight_tensor = Tensor(data_layout=types_pb2.NC,
tensor_data=np.random.rand(
10, 32768).astype(np.float16))
bn_mean_tensor = Tensor(data_layout=types_pb2.NC,
tensor_data=np.random.rand(1, 64).astype(
np.float16))
bn_var_tensor = Tensor(data_layout=types_pb2.NC,
tensor_data=np.random.rand(1, 64).astype(
np.float16))
bn_gamma_tensor = Tensor(data_layout=types_pb2.NC,
tensor_data=np.random.rand(1, 64).astype(
np.float16))
bn_beta_tensor = Tensor(data_layout=types_pb2.NC,
tensor_data=np.random.rand(1, 64).astype(
np.float16))
act = data_op.input_data(input_tensor, "input")
act = nn_ops.convolution(act,
filter_tensor,
stride=[1, 1],
padding="same",
activation=None,
name="conv_none")
act = nn_ops.convolution(act,
filter_tensor,
stride=[1, 1],
padding="same",
activation="relu",
name="conv_relu")
act = nn_ops.convolution(act,
filter_tensor,
stride=[1, 1],
padding="same",
activation="lrelu",
name="conv_lrelu")
act = nn_ops.convolution(act,
filter_tensor,
stride=[1, 1],
padding="same",
activation="elu",
name="conv_elu")
act = nn_ops.convolution(act,
filter_tensor,
stride=[1, 1],
padding="same",
activation="selu",
name="conv_selu")
act = nn_ops.convolution(act,
filter_tensor,
stride=[1, 1],
padding="same",
activation="tanh",
name="conv_tanh")
act = nn_ops.convolution(act,
filter_tensor,
stride=[1, 1],
padding="same",
activation="hard_tanh",
name="conv_hard_tanh")
act = nn_ops.convolution(act,
filter_tensor,
stride=[1, 1],
padding="same",
activation="sigmoid",
name="conv_sigmoid")
act = nn_ops.convolution(act,
filter_tensor,
stride=[1, 1],
padding="same",
activation="softmax",
name="conv_softmax")
act = nn_ops.batch_norm(act,
bn_mean_tensor,
bn_var_tensor,
bn_gamma_tensor,
bn_beta_tensor,
activation="relu",
name="bn_relu")
out = nn_ops.mat_mul(act,
weight_tensor,
activation="relu",
name="fc_relu")
out = nn_ops.mat_mul(act,
weight_tensor,
activation="lrelu",
activation_params={"slope": 0.1},
name="fc_lrelu")
out = nn_ops.mat_mul(act,
weight_tensor,
activation="elu",
activation_params={"alpha": 0.3},
name="fc_elu")
out = nn_ops.mat_mul(act,
weight_tensor,
activation="selu",
activation_params={
"alpha": 1.0,
"lambda_param": 1.0
},
name="fc_selu")
out = nn_ops.mat_mul(act,
weight_tensor,
activation="hard_tanh",
activation_params={
"min": -2.0,
"max": 2.0
},
name="fc_hard_tanh")
graph_proto, _ = test_graph.to_proto()
# None
node = get_node_proto(graph_proto, "conv_none")
self.assertEqual(node.params.act_params.activation,
types_pb2.UnknownOp)
# ReLU
node = get_node_proto(graph_proto, "conv_relu")
self.assertEqual(node.params.act_params.activation, types_pb2.ReLU)
# LReLU (default slope = 0.2)
node = get_node_proto(graph_proto, "conv_lrelu")
self.assertEqual(node.params.act_params.activation, types_pb2.LReLU)
self.assertAlmostEqual(node.params.act_params.lrelu_params.slope, 0.2)
# ELU (default alpha = 0.1)
node = get_node_proto(graph_proto, "conv_elu")
self.assertEqual(node.params.act_params.activation, types_pb2.ELU)
self.assertAlmostEqual(node.params.act_params.elu_params.alpha, 0.1)
# SELU (default alpha = 1.6733, lambda = 1.0507)
node = get_node_proto(graph_proto, "conv_selu")
self.assertEqual(node.params.act_params.activation, types_pb2.SELU)
self.assertAlmostEqual(node.params.act_params.elu_params.alpha, 1.6733,
5)
self.assertAlmostEqual(node.params.act_params.elu_params.lambda_param,
1.0507, 5)
# Tanh
node = get_node_proto(graph_proto, "conv_tanh")
self.assertEqual(node.params.act_params.activation, types_pb2.Tanh)
# HardTanh (default min = -1, max = 1)
node = get_node_proto(graph_proto, "conv_hard_tanh")
self.assertEqual(node.params.act_params.activation, types_pb2.HardTanh)
self.assertAlmostEqual(node.params.act_params.hard_tanh_params.min, -1)
self.assertAlmostEqual(node.params.act_params.hard_tanh_params.max, 1)
# Sigmoid
node = get_node_proto(graph_proto, "conv_sigmoid")
self.assertEqual(node.params.act_params.activation, types_pb2.Sigmoid)
# Softmax
node = get_node_proto(graph_proto, "conv_softmax")
self.assertEqual(node.params.act_params.activation, types_pb2.Softmax)
# Fusion with inner products and batch norms.
node = get_node_proto(graph_proto, "bn_relu")
self.assertEqual(node.params.act_params.activation, types_pb2.ReLU)
node = get_node_proto(graph_proto, "fc_relu")
self.assertEqual(node.params.act_params.activation, types_pb2.ReLU)
node = get_node_proto(graph_proto, "fc_lrelu")
self.assertEqual(node.params.act_params.activation, types_pb2.LReLU)
self.assertAlmostEqual(node.params.act_params.lrelu_params.slope, 0.1)
node = get_node_proto(graph_proto, "fc_elu")
self.assertEqual(node.params.act_params.activation, types_pb2.ELU)
self.assertAlmostEqual(node.params.act_params.elu_params.alpha, 0.3)
node = get_node_proto(graph_proto, "fc_selu")
self.assertEqual(node.params.act_params.activation, types_pb2.SELU)
self.assertAlmostEqual(node.params.act_params.elu_params.alpha, 1.0)
self.assertAlmostEqual(node.params.act_params.elu_params.lambda_param,
1.0)
node = get_node_proto(graph_proto, "fc_hard_tanh")
self.assertEqual(node.params.act_params.activation, types_pb2.HardTanh)
self.assertAlmostEqual(node.params.act_params.hard_tanh_params.min,
-2.0)
self.assertAlmostEqual(node.params.act_params.hard_tanh_params.max,
2.0)