def testConvolutionsMatchFwdBwdWuVariableLR(self):
with ops.device("/device:IPU:0"):
x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2])
lr = array_ops.placeholder(np.float32, shape=[])
with variable_scope.variable_scope("vs", use_resource=True):
y = layers.Conv2D(
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer(),
name='conv1')(x)
y = layers.Conv2D(
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer(),
name='conv2')(y)
y = layers.Conv2D(
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer(),
name='conv3')(y)
loss = math_ops.reduce_sum(y)
optimizer = gradient_descent.GradientDescentOptimizer(lr)
train = optimizer.minimize(loss)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system(True, True, True)
with tu.ipu_session() as sess:
sess.run(variables.global_variables_initializer())
sess.run(report)
sess.run([train, loss], {x: np.zeros([1, 4, 4, 2]), lr: 0.1})
result = sess.run(report)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
# Fwd and BackpropInput should be shared
# Weight transpose for BackpropInput should be present
# Both BackpropFilter should be shared
ok = [
'__seed*', 'host-exchange-local-copy-', 'Copy_',
'vs/conv1/Conv2D/convolution.*/Conv_1x1',
'Sum/reduce.*/ReduceFinalStage/IntermediateToOutput/Reduce',
'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*/Conv_4x4',
'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*/AddTo'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testBatchNormAndGroupNormalizeMixedInference(self):
with ops.device("/device:IPU:0"):
x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2])
with variable_scope.variable_scope("vs", use_resource=True):
y = convolutional.conv2d(
x,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer())
gamma = constant_op.constant([0.5, 0.5], np.float32)
beta = constant_op.constant([0.5, 0.5], np.float32)
mean = constant_op.constant([0.5, 0.5], np.float32)
inv_std_dev = constant_op.constant([0.5, 0.5], np.float32)
y = gen_popnn_ops.popnn_group_norm_inference(
inputs=y,
gamma=gamma,
beta=beta,
mean=mean,
inv_std_dev=inv_std_dev,
data_format="NHWC",
epsilon=0.0015,
num_groups=2)
y = convolutional.conv2d(
y,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer())
y = layers_norm.batch_normalization(y, fused=True)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system(True, True, True)
with tu.ipu_session() as sess:
sess.run(variables.global_variables_initializer())
sess.run(report)
sess.run(y, {x: np.zeros([1, 4, 4, 2])})
result = sess.run(report)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
# Would fail if there were two batch norms in the graph
ok = [
'__seed*', 'host-exchange-local-copy', 'Copy_',
'vs/conv2d/Conv2D/convolution.*/Conv_1x1/Convolve',
'vs/PopnnGroupNormInference/custom-call*/',
'vs/batch_normalization/FusedBatchNorm/batch-norm-inference.*/'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testFwdAndBwdMaxPool(self):
input = np.arange(16).reshape(1, 4, 4, 1)
output_grad = np.full((1, 2, 2, 1), 0.1)
with ops.device("/device:IPU:0"):
pa = array_ops.placeholder(np.float32, [1, 4, 4, 1], name="a")
pb = array_ops.placeholder(np.float32, [1, 2, 2, 1], name="b")
c = nn.max_pool(pa,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
data_format='NCHW',
padding='SAME')
d = gen_nn_ops.max_pool_grad(pa,
c,
pb,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
data_format='NCHW',
padding='SAME')
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system()
with tu.ipu_session() as sess:
sess.run(report)
fe = {
pa: input,
pb: output_grad,
}
output, input_grad = sess.run((c, d), fe)
self.assertAllClose(output, [[[[5.], [7.]], [[13.], [15.]]]])
self.assertAllClose(
input_grad,
[[[[0.], [0.], [0.], [0.]], [[0.], [0.1], [0.], [0.1]],
[[0.], [0.], [0.], [0.]], [[0.], [0.1], [0.], [0.1]]]])
result = sess.run(report)
self.assertTrue(len(result) == 3)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*', 'Copy_*', 'MaxPool/custom-call*/maxPool2x2/',
'MaxPoolGrad/custom-call*/maxPool2x2'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testWideConstantWithAllocationTarget(self):
# This test will fail if the dynamic slice is not mapped correctly.
dtype = np.float32
shape = (512, 2, 2048)
def my_net(y):
def cond(i, x, y):
return i < 2
def body(i, x, y):
s = array_ops.slice(x, [i, i, i], [1, 1, 2048])
y = y + math_ops.reduce_mean(s)
i = i + 1
return (i, x, y)
i = 0
c = constant_op.constant(4, shape=shape, dtype=dtype, name="c")
return control_flow_ops.while_loop(cond, body, (i, c, y))[2]
with ops.device('cpu'):
y = array_ops.placeholder(dtype, [1])
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system()
with ops.device("/device:IPU:0"):
r = xla.compile(my_net, inputs=[y])
with tu.ipu_session() as sess:
sess.run(report)
y = sess.run(r, {y: [10]})
self.assertAllClose(y[0], [18])
result = sess.run(report)
self.assertTrue(len(result) == 3)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*', 'Copy_*_to_*',
'while/Slice/dynamic-slice*/dynamicSlice', 'while/Mean/reduce',
'while/Mean/multiply', 'while/add*/add*/AddTo'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
max_tile_size = tu.get_maximum_tile_size_from_events(s)
self.assertTrue(max_tile_size < 60000)
def testBatchNormalizeInferenceMatchWithSharding(self):
with ops.device("/device:IPU:0"):
x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2])
with variable_scope.variable_scope("vs", use_resource=True):
with tu.ipu_shard(0):
a = convolutional.conv2d(
x,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer())
b = layers_norm.batch_normalization(a, fused=True)
with tu.ipu_shard(0):
c = convolutional.conv2d(
b,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer())
d = layers_norm.batch_normalization(c, fused=True)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system(True, True, True, sharded=True)
with tu.ipu_session() as sess:
sess.run(variables.global_variables_initializer())
sess.run(report)
sess.run(d, {x: np.zeros([1, 4, 4, 2])})
result = sess.run(report)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
# Would fail if there were two batch norms in the graph
ok = [
'__seed*', '*OnTileCopy*',
'vs/conv2d/Conv2D/convolution.*/Conv_1x1/Convolve',
'vs/batch_normalization/FusedBatchNorm/batch-norm-inference.*/'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testBatchNormalizeInferenceDontMatchDifferentTypes(self):
with ops.device("/device:IPU:0"):
x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2])
with variable_scope.variable_scope("vs", use_resource=True):
y = convolutional.conv2d(
x,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer())
y = layers_norm.batch_normalization(y, fused=True)
y = math_ops.cast(y, np.float16)
y = convolutional.conv2d(
y,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer())
y = layers_norm.batch_normalization(y, fused=True)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system(True, True, True)
with tu.ipu_session() as sess:
sess.run(variables.global_variables_initializer())
sess.run(report)
sess.run(y, {x: np.zeros([1, 4, 4, 2])})
result = sess.run(report)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
# Matches two convolutions
ok = [
'__seed*', 'host-exchange-local-copy-', 'Copy_',
'vs/conv2d/Conv2D/convolution.*/Conv_1x1',
'vs/batch_normalization/FusedBatchNorm/batch-norm-inference.*/',
'vs/Cast/convert.*/Cast',
'vs/conv2d_1/Conv2D/convolution.*/Conv_1x1',
'vs/batch_normalization_1/FusedBatchNormV2/batch-norm-inference.*/'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testAvgPoolSameWithReshape(self):
np.random.seed(0)
shape = [1, 10, 10, 1]
data = np.random.uniform(0, 1, shape)
# The expected answer was generated using TF on the cpu
expected = [[[[0.64431685], [0.51738459], [0.49705142], [0.60235918],
[0.73694557]],
[[0.57755166], [0.47387227], [0.40451217], [0.4876942],
[0.55843753]],
[[0.49037799], [0.4466258], [0.35829377], [0.40070742],
[0.37205362]],
[[0.47563809], [0.4075647], [0.34894851], [0.35470542],
[0.3322109]],
[[0.52914065], [0.45464769], [0.38156652], [0.32455513],
[0.33199897]]]]
with ops.device("/device:IPU:0"):
pa = array_ops.placeholder(np.float32, shape, name="a")
output = nn.avg_pool(pa,
ksize=[1, 5, 5, 1],
strides=[1, 2, 2, 1],
data_format='NHWC',
padding='SAME',
name="avg")
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system(True, True, True)
with tu.ipu_session() as sess:
sess.run(variables.global_variables_initializer())
sess.run(report)
fd = {pa: data}
result = sess.run(output, fd)
self.assertAllClose(result, expected)
result = sess.run(report)
self.assertEqual(len(result), 4)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = ['__seed*', 'avg/custom-call*/avgPool5x5']
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConvolutionBiasApply(self):
with ops.device("/device:IPU:0"):
x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2])
with variable_scope.variable_scope("vs", use_resource=True):
y = layers.Conv2D(
2,
1,
use_bias=True,
kernel_initializer=init_ops.ones_initializer())(x)
y = layers.Conv2D(
2,
1,
use_bias=True,
kernel_initializer=init_ops.ones_initializer())(y)
loss = math_ops.reduce_sum(y)
optimizer = gradient_descent.GradientDescentOptimizer(0.1)
train = optimizer.minimize(loss)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system(True, True, True)
with tu.ipu_session() as sess:
sess.run(variables.global_variables_initializer())
sess.run(report)
sess.run([train, loss], {x: np.zeros([1, 4, 4, 2])})
result = sess.run(report)
self.assertEqual(
len(result),
6) # 2xcompile, 1xupload, 1xload, 1xdownload, 1xexecute
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*',
'GradientDescent/update_vs/conv2d/bias/ResourceApplyGradientDescent/fusion.*/Reduce'
]
self.assertTrue(
tu.check_compute_sets_in_whitelist_entries(cs_list, ok))
def testBatchNormalizeFused(self):
x = array_ops.placeholder(np.float32, [4, 64, 64, 4], name="a")
with ops.device("/device:IPU:0"):
with variable_scope.variable_scope("", use_resource=True):
beta = variable_scope.get_variable(
"x",
dtype=np.float32,
shape=[4],
initializer=init_ops.constant_initializer(0.0))
gamma = variable_scope.get_variable(
"y",
dtype=np.float32,
shape=[4],
initializer=init_ops.constant_initializer(1.0))
b_mean, b_var = nn.moments(x, [0, 1, 2], name='moments')
normed = nn.fused_batch_norm(x,
gamma,
beta,
b_mean,
b_var,
is_training=False)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system()
with tu.ipu_session() as sess:
sess.run(report)
sess.run(variables.global_variables_initializer())
result, _, _ = sess.run(normed, {x: np.zeros([4, 64, 64, 4])})
self.assertAllClose(result, np.zeros([4, 64, 64, 4]))
rep = sess.run(report)
s = tu.extract_all_strings_from_event_trace(rep)
cs = tu.get_compute_sets_from_report(s)
bl = ['*convert*/Cast*']
self.assertTrue(tu.check_compute_sets_not_in_blacklist(cs, bl))
def testConv8x8_WithBias(self):
for fmt in self.data_formats:
with ops.device("/device:IPU:0"):
inp = array_ops.placeholder(
np.float32, self._ip_shp([1, 84, 84, 4], fmt), name="inp")
wei = array_ops.placeholder(np.float32, [8, 8, 4, 16], name="wei")
bia = array_ops.placeholder(np.float32, [16], name="bia")
output = nn_ops.conv2d(
inp,
wei,
strides=self._ip_shp([1, 4, 4, 1], fmt),
padding="VALID",
data_format=fmt,
name='cnv4')
output = nn_ops.bias_add(output, bia, data_format=fmt, name='ba4')
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system()
with tu.ipu_session() as sess:
sess.run(report)
fd = {
inp: np.zeros(self._ip_shp([1, 84, 84, 4], fmt)),
wei: np.zeros([8, 8, 4, 16]),
bia: np.zeros([16]),
}
result = sess.run(output, fd)
self.assertAllClose(result, np.zeros(self._ip_shp([1, 20, 20, 16], fmt)))
result = sess.run(report)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*', 'host-exchange-local-copy-',
'Copy_XLA_Args/arg2.*_weights_to_cnv4*/convolution.*/Conv_8x8_stride4x4/weightsRearranged',
'cnv4*/convolution.*/Conv_8x8_stride4x4', 'ba4*/fusion/addToChannel'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def tesInplaceAddCopyWithInplacePeer2(self):
data_a = np.array([[10, -10], [-5, 5]])
data_b = np.array([[-15, 15], [25, -25]])
data_c = 2
with ops.device("/device:IPU:0"):
pa = array_ops.placeholder(np.float32, [2, 2])
pb = array_ops.placeholder(np.float32, [2, 2])
pc = array_ops.placeholder(np.float32, [])
a = array_ops.transpose(pa)
b = pa + pb * pc
c = a * pb + pc
d = b / c
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system()
with tu.ipu_session() as sess:
sess.run(report)
fd = {
pa: data_a,
pb: data_b,
pc: data_c,
}
np_result = (data_a + data_b * data_c) / (
np.transpose(data_a) * data_b + data_c)
result = sess.run(d, fd)
self.assertAllClose(result, np_result)
result = sess.run(report)
self.assertTrue(len(result) == 3) #compile_begin, compile_end, execute
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*', 'Copy_XLA_Args/arg0.*_to_transpose/transpose'
'mul/multiply.*/Op/Multiply', 'add/add.*/AddTo',
'mul_1/multiply.*/Op/Multiply', 'add_1/add.*/AddTo',
'truediv/divide.*/Op/Divide'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConvolutionsDontMatchDifferentDevices(self):
with ops.device("/device:IPU:0"):
x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2])
with variable_scope.variable_scope("vs", use_resource=True):
with tu.ipu_shard(0):
y = layers.Conv2D(
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer())(x)
with tu.ipu_shard(1):
y = layers.Conv2D(
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer())(y)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system(True, True, True, sharded=True)
with tu.ipu_session() as sess:
sess.run(variables.global_variables_initializer())
sess.run(report)
sess.run(y, {x: np.zeros([1, 4, 4, 2])})
result = sess.run(report)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
# Note how there are two convolutions
ok = [
'__seed*', '*OnTileCopy*', 'vs/conv2d/Conv2D/convolution.*',
'Copy_vs/conv2d/Conv2D/convolution.*',
'vs/conv2d_1/Conv2D/convolution.*'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConvolutionsDontMatchDifferentConvParams(self):
with ops.device("/device:IPU:0"):
x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2])
with variable_scope.variable_scope("vs", use_resource=True):
y = layers.Conv2D(
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer())(x)
y = layers.Conv2D(
2,
1,
use_bias=False,
strides=(2, 1),
kernel_initializer=init_ops.ones_initializer())(y)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system(True, True, True)
with tu.ipu_session() as sess:
sess.run(variables.global_variables_initializer())
sess.run(report)
sess.run(y, {x: np.zeros([1, 4, 4, 2])})
result = sess.run(report)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
# Matches two convolutions
ok = [
'__seed*', 'Copy_*weightsRearranged',
'host-exchange-local-copy-',
'vs/conv2d/Conv2D/convolution.*/Conv_1x1',
'vs/conv2d_1/Conv2D/convolution.*/Conv_1x1'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConvWithBnAndRelu(self):
with ops.device("/device:IPU:0"):
x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2])
with variable_scope.variable_scope("vs", use_resource=True):
y = layers.Conv2D(
2,
1,
use_bias=True,
kernel_initializer=init_ops.ones_initializer())(x)
y = layers_norm.batch_normalization(y, fused=True)
y = nn_ops.relu(y)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system(True, True, True)
with tu.ipu_session() as sess:
sess.run(variables.global_variables_initializer())
sess.run(report)
sess.run(y, {x: np.zeros([1, 4, 4, 2])})
result = sess.run(report)
self.assertEqual(
len(result),
6) # 2xcompile, 1xupload 1xload, 1xdownload, 1xexecute
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*', 'host-exchange-local-copy', 'Copy_',
'vs/conv2d/Conv2D/convolution.*/Conv_1x1', 'vs/conv2d/BiasAdd',
'vs/batch_normalization/FusedBatchNorm/batch-norm-inference.*/',
'vs/Relu/custom-call/Nonlinearity'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testAvgPoolValidWithBroadcast(self):
np.random.seed(0)
shape = [1, 10, 10, 1]
data = np.random.uniform(0, 1, shape)
# The expected answer was generated using TF on the cpu
expected = [[[[0.52647954], [0.44196457], [0.49284577]],
[[0.44039682], [0.44067329], [0.44934618]],
[[0.46444583], [0.45419583], [0.38236427]]]]
with ops.device("/device:IPU:0"):
pa = array_ops.placeholder(np.float32, shape, name="a")
output = nn.avg_pool(pa,
ksize=[1, 5, 5, 1],
strides=[1, 2, 2, 1],
data_format='NHWC',
padding='VALID',
name="avg")
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system(True, True, True)
with tu.ipu_session() as sess:
sess.run(variables.global_variables_initializer())
sess.run(report)
fd = {pa: data}
result = sess.run(output, fd)
self.assertAllClose(result, expected)
result = sess.run(report)
self.assertEqual(len(result), 4)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = ['__seed*', 'avg/custom-call*/avgPool5x5']
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def test3DConv8x8x8_WithBias(self):
with ops.device("/device:IPU:0"):
inp = array_ops.placeholder(np.float32, [1, 84, 84, 84, 2],
name="inp")
wei = array_ops.placeholder(np.float32, [8, 8, 8, 2, 4],
name="wei")
bia = array_ops.placeholder(np.float32, [4], name="bia")
output = nn_ops.conv3d(inp,
wei,
strides=[1, 4, 4, 4, 1],
padding="VALID")
output = nn_ops.bias_add(output, bia)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system()
with tu.ipu_session() as sess:
sess.run(report)
fd = {
inp: np.zeros([1, 84, 84, 84, 2]),
wei: np.zeros([8, 8, 8, 2, 4]),
bia: np.zeros([4]),
}
result = sess.run(output, fd)
self.assertAllClose(result, np.zeros([1, 20, 20, 20, 4]))
result = sess.run(report)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*', 'host-exchange-local-copy-', 'Copy_',
'Conv3D/convolution.*/Conv_8x8x8_stride4x4x4',
'BiasAdd/fusion/addToChannel'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testInplaceTuple(self):
def my_net(x):
def cond(i, x, y):
return i < 1
def body(i, x, y):
i = i + 1
x = nn.tanh(x)
y = nn.tanh(y)
return (i, x, y)
i = 0
return control_flow_ops.while_loop(cond, body, (i, x, x))[1:]
with ops.device('cpu'):
x = array_ops.placeholder(np.float32, [4])
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system()
with ops.device("/device:IPU:0"):
r = xla.compile(my_net, inputs=[x])
with tu.ipu_session() as sess:
sess.run(report)
x, y = sess.run(r, {x: np.full([4], 2)})
self.assertAllClose(x, np.full([4], np.tanh(2)))
self.assertAllClose(y, np.full([4], np.tanh(2)))
result = sess.run(report)
self.assertTrue(len(result) == 3)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*', 'Copy_*_to_*', 'while/Tanh/tanh*/Op/Tanh',
'while/Tanh_1/tanh*/Op/Tanh'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConv3x3_WithBias(self):
for fmt in self.data_formats:
with ops.device("/device:IPU:0"):
pa = array_ops.placeholder(
np.float32, self._ip_shp([1, 14, 14, 64], fmt), name="a")
pb = array_ops.placeholder(np.float32, [3, 3, 64, 128], name="b")
bi = array_ops.placeholder(np.float32, [128], name="b")
output = nn_ops.convolution(
pa, pb, padding="SAME", data_format=fmt, name='cnv3')
output = nn_ops.bias_add(output, bi, data_format=fmt, name='ba3')
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system()
with tu.ipu_session() as sess:
sess.run(report)
fd = {
pa: np.zeros(self._ip_shp([1, 14, 14, 64], fmt)),
pb: np.zeros([3, 3, 64, 128]),
bi: np.zeros([128]),
}
result = sess.run(output, fd)
self.assertAllClose(result, np.zeros(
self._ip_shp([1, 14, 14, 128], fmt)))
result = sess.run(report)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*', 'Copy_*actsRearranged', 'host-exchange-local-copy-',
'cnv3*/convolution.*/Conv_3x3', 'ba3*/fusion/addToChannel'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def test3DConvBackpropInput(self):
with ops.device("/device:IPU:0"):
ins = constant_op.constant([2, 8, 8, 8, 3], np.int32)
fil = array_ops.placeholder(np.float32, [2, 2, 2, 3, 5],
name="inp")
bck = array_ops.placeholder(np.float32, [2, 8, 8, 8, 5],
name="wei")
output = nn_ops.conv3d_backprop_input_v2(ins,
fil,
bck,
strides=[1, 1, 1, 1, 1],
padding="SAME")
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system()
with tu.ipu_session() as sess:
sess.run(report)
fd = {
fil: np.zeros([2, 2, 2, 3, 5]),
bck: np.zeros([2, 8, 8, 8, 5]),
}
result = sess.run(output, fd)
self.assertAllClose(result, np.zeros([2, 8, 8, 8, 3]))
result = sess.run(report)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*', 'Copy_', 'Conv3DBackpropInputV2/fusion*/Conv_2x2x2'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testFullyConnectedWithBias(self):
with ops.device("/device:IPU:0"):
x = array_ops.placeholder(np.float32, shape=[2, 2])
weights = array_ops.placeholder(np.float32, shape=[2, 2])
bias = array_ops.placeholder(np.float32, shape=[2])
x_new = nn.xw_plus_b(x, weights, bias)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system(True, True, True)
with tu.ipu_session() as sess:
sess.run(report)
out = sess.run(
x_new, {
x: np.full([2, 2], 3),
weights: np.full([2, 2], 4),
bias: np.ones([2]),
})
self.assertAllClose(np.full([2, 2], 25), out)
result = sess.run(report)
self.assertEqual(len(result),
4) # 1xcompile, 1xload, 1xdownload, 1xexecute
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*', 'host-exchange-local-copy',
'xw_plus_b/MatMul/dot.*/Conv_1/Convolve',
'xw_plus_b/fusion/addToChannel'
]
self.assertTrue(
tu.check_compute_sets_in_whitelist_entries(cs_list, ok))
def testInplaceOpAddCopyWithInplaceParent(self):
with ops.device("/device:IPU:0"):
pa = array_ops.placeholder(np.float32, [3])
pb = array_ops.placeholder(np.float32, [3])
pc = array_ops.placeholder(np.float32, [])
c = array_ops.slice(pa, [0], [2])
d = array_ops.slice(pb, [0], [2])
e = c + d
f = e / pc
g = array_ops.slice(pa, [1], [2])
h = f + g
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system()
with tu.ipu_session() as sess:
sess.run(report)
fd = {
pa: [1, 2, 3],
pb: [5, 6, 7],
pc: 2,
}
result = sess.run(h, fd)
self.assertAllClose(result, [5, 7])
result = sess.run(report)
self.assertTrue(len(result) == 3)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*', 'Copy_XLA_Args/arg*_to_Slice*/slice*.clone',
'add/add.*/AddTo', 'truediv/divide.*/Op/Divide', 'add_1/add.*/AddTo'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testRelu(self):
with ops.device("/device:IPU:0"):
pa = array_ops.placeholder(np.float32, [3], name="a")
c = nn_ops.relu(pa)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system()
with tu.ipu_session() as sess:
fd = {pa: [-6.0, 0.0, 6.0]}
result = sess.run(c, fd)
self.assertAllClose(result, [0.0, 0.0, 6.0])
result = sess.run(report)
self.assertTrue(len(result) == 3)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = ['__seed*', 'Relu/custom-call/Nonlinearity']
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testDepthwiseConv3x1(self):
with ops.device("/device:IPU:0"):
pa = array_ops.placeholder(np.float32, [1, 2, 2, 3], name="a")
pb = array_ops.placeholder(np.float32, [1, 1, 3, 1], name="b")
pc = array_ops.placeholder(np.float32, [3], name="c")
c = nn.depthwise_conv2d(pa, pb, strides=[1, 1, 1, 1], padding="SAME")
output = c + pc
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system()
with tu.ipu_session() as sess:
sess.run(report)
fd = {
pa: [[[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]],
pb: [[[[6], [4], [2]]]],
pc: [1, 1, 1]
}
result = sess.run(output, fd)
self.assertAllClose(
result, [[[[7, 9, 7], [25, 21, 13]], [[43, 33, 19], [61, 45, 25]]]])
result = sess.run(report)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*', 'host-exchange-local-copy-', 'Copy_',
'depthwise/convolution.*/Conv_1x1',
'Copy_depthwise/convolution.*/Conv_1x1/partials_to_depthwise/convolution.*/Conv_1x1/partials[[]cloned[]]',
'add/fusion*/addToChannel'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testTruncatedNormalInitalizer(self):
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
with ops.device("/device:IPU:0"):
with variable_scope.variable_scope("", use_resource=True):
i = init_ops.truncated_normal_initializer(mean=1.0,
stddev=0.01)
z = variable_scope.get_variable("z1",
shape=[2, 4],
dtype=np.float32,
initializer=i)
tu.configure_ipu_system()
with tu.ipu_session() as sess:
# Clean existing reports
sess.run(report)
sess.run(variables.global_variables_initializer())
o = sess.run(z)
self.assertAllClose(o, np.ones((2, 4)), 0.2, 0.2)
# Find of the names of compute sets
r = sess.run(report)
s = tu.extract_all_strings_from_event_trace(r)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*',
'z1/Initializer/truncated_normal/TruncatedNormal/custom-call*/truncatedNormal',
'z1/Initializer/truncated_normal/mul/multiply.*/Op/Multiply',
'z1/Initializer/truncated_normal/add.*/AddTo'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def tesInplaceAddCopyWithInplacePeer(self):
data_a = np.array([[10, -20], [5, 1]])
data_b = np.array([[-12, 11], [12, -13]])
with ops.device("/device:IPU:0"):
pa = array_ops.placeholder(np.float32, [2, 2])
pb = array_ops.placeholder(np.float32, [2, 2])
c = array_ops.transpose(pa)
d = pa + pb
e = c / d
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system()
with tu.ipu_session() as sess:
sess.run(report)
fd = {
pa: data_a,
pb: data_b,
}
result = sess.run(e, fd)
np_result = np.transpose(data_a) / (data_a + data_b)
self.assertAllClose(result, np_result)
result = sess.run(report)
self.assertTrue(len(result) == 3) #compile_begin, compile_end, execute
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*', 'host-exchange-local-copy-',
'Copy_XLA_Args/arg0.*_to_transpose/transpose', 'add/add.*/AddTo',
'truediv/divide.*/Op/Divide'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def test3DConv3x3x3_WithBias(self):
with ops.device("/device:IPU:0"):
pa = array_ops.placeholder(np.float32, [1, 14, 14, 14, 16],
name="a")
pb = array_ops.placeholder(np.float32, [3, 3, 3, 16, 32], name="b")
bi = array_ops.placeholder(np.float32, [32], name="b")
output = nn_ops.convolution(pa, pb, padding="SAME")
output = nn_ops.bias_add(output, bi)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system()
with tu.ipu_session() as sess:
sess.run(report)
fd = {
pa: np.zeros([1, 14, 14, 14, 16]),
pb: np.zeros([3, 3, 3, 16, 32]),
bi: np.zeros([32]),
}
result = sess.run(output, fd)
self.assertAllClose(result, np.zeros([1, 14, 14, 14, 32]))
result = sess.run(report)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = [
'__seed*', 'host-exchange-local-copy-', 'Copy_',
'convolution/convolution.*/Conv_3x3x3',
'BiasAdd/fusion/addToChannel'
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testBatchNormsMatchFwdBwd(self):
with ops.device("/device:IPU:0"):
x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2])
with variable_scope.variable_scope("vs", use_resource=True):
y = convolutional.conv2d(
x,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer(),
name='conv1')
y = layers_norm.batch_normalization(y,
fused=True,
training=True)
y = convolutional.conv2d(
y,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer(),
name='conv2')
y = layers_norm.batch_normalization(y,
fused=True,
training=True)
y = convolutional.conv2d(
y,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer(),
name='conv3')
y = layers_norm.batch_normalization(y,
fused=True,
training=True)
loss = math_ops.reduce_sum(y)
optimizer = gradient_descent.GradientDescentOptimizer(0.1)
train = optimizer.minimize(loss)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system(True, True, True)
with tu.ipu_session() as sess:
sess.run(variables.global_variables_initializer())
sess.run(report)
sess.run([train, loss], {x: np.zeros([1, 4, 4, 2])})
result = sess.run(report)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
# One BN for forwards and one BN for grad
# (note that we don't cache gradient application)
ok = [
'__seed*',
'host-exchange-local-copy-',
'Copy_',
'vs/conv1/Conv2D/convolution.*/Conv_1x1',
'vs/batch_normalization/FusedBatchNorm/batch-norm-training.*/',
'Sum/reduce.*/ReduceFinalStage/IntermediateToOutput/Reduce',
'gradients/vs/batch_normalization_2/FusedBatchNorm_grad/FusedBatchNormGrad/batch-norm-grad.*/',
'GradientDescent/update_vs/batch_normalization/',
'GradientDescent/update_vs/batch_normalization_1/',
'GradientDescent/update_vs/batch_normalization_2/',
'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*/Conv_4x4/Convolve',
'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*/AddTo',
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testGroupNormsMatchFwdBwd(self):
with ops.device("/device:IPU:0"):
x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2])
with variable_scope.variable_scope("vs", use_resource=True):
y = convolutional.conv2d(
x,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer(),
name='conv1')
gamma = constant_op.constant([0.5, 0.5], np.float32)
beta = constant_op.constant([0.5, 0.5], np.float32)
y, _, _ = gen_popnn_ops.popnn_group_norm_training(
inputs=y,
gamma=gamma,
beta=beta,
data_format="NHWC",
epsilon=0.0015,
num_groups=2)
y = convolutional.conv2d(
y,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer(),
name='conv2')
y, _, _ = gen_popnn_ops.popnn_group_norm_training(
inputs=y,
gamma=gamma,
beta=beta,
data_format="NHWC",
epsilon=0.0015,
num_groups=2)
y = convolutional.conv2d(
y,
2,
1,
use_bias=False,
kernel_initializer=init_ops.ones_initializer(),
name='conv3')
y, _, _ = gen_popnn_ops.popnn_group_norm_training(
inputs=y,
gamma=gamma,
beta=beta,
data_format="NHWC",
epsilon=0.0015,
num_groups=2)
loss = math_ops.reduce_sum(y)
optimizer = gradient_descent.GradientDescentOptimizer(0.1)
train = optimizer.minimize(loss)
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system(True, True, True)
with tu.ipu_session() as sess:
sess.run(variables.global_variables_initializer())
sess.run(report)
sess.run([train, loss], {x: np.zeros([1, 4, 4, 2])})
result = sess.run(report)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
# One GN for forwards and one GN for grad
ok = [
'__seed*',
'host-exchange-local-copy-',
'Copy_',
'vs/conv1/Conv2D/convolution*/Conv_1x1/Convolve',
'vs/PopnnGroupNormTraining/custom-call*/Norm',
'vs/PopnnGroupNormTraining/custom-call*/iStdDev',
'vs/PopnnGroupNormTraining/custom-call*/Whiten',
'Sum/reduce.*/*/Reduce',
'gradients/vs/PopnnGroupNormTraining_2_grad/PopnnGroupNormGrad/custom-call*/',
'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*',
]
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
