def testMaxPool(self): with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, [1, 1, 10, 10], name="a") c = nn.max_pool(pa, ksize=[1, 1, 5, 5], strides=[1, 1, 2, 2], data_format='NCHW', padding='SAME', name="max") with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = { pa: np.ones([1, 1, 10, 10]), } result = sess.run(c, fd) self.assertAllClose(result, np.ones([1, 1, 5, 5])) result = sess.run(report) self.assertTrue(len(result) == 3) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = ['__seed*', 'max/custom-call*/maxPool5x5'] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConvBackpropFilter(self): with ops.device("/device:IPU:0"): inp = array_ops.placeholder(np.float32, [2, 8, 8, 3]) fil = constant_op.constant([2, 2, 3, 5], np.int32) bck = array_ops.placeholder(np.float32, [2, 8, 8, 5], name="wei") output = nn_ops.conv2d_backprop_filter( inp, fil, bck, strides=[1, 1, 1, 1], padding="SAME") with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = { inp: np.zeros([2, 8, 8, 3]), bck: np.zeros([2, 8, 8, 5]), } result = sess.run(output, fd) self.assertAllClose(result, np.zeros([2, 2, 3, 5])) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = ['__seed*', 'Copy_', 'Conv2DBackpropFilter/convolution.*/Conv_8x8'] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testSigmoidNotInplace(self): with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, [3], name="a") c = math_ops.sigmoid(pa) + pa with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = {pa: [-6.0, 0.0, 6.0]} result = sess.run(c, fd) self.assertAllClose(result, [-5.997527, 0.5, 6.997527]) result = sess.run(report) self.assertTrue(len(result) == 3) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'Sigmoid/custom-call/Nonlinearity', 'Copy_XLA_Args/arg0.*_to_Sigmoid/custom-call.clone/OnTileCopy-0', 'add/add.*/AddTo' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testRandomNormalInitalizer(self): with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() with ops.device("/device:IPU:0"): with variable_scope.variable_scope("vs", use_resource=True): i = init_ops.random_normal_initializer(mean=2.0, stddev=0.01) z = variable_scope.get_variable("z1", shape=[], dtype=np.float32, initializer=i) tu.configure_ipu_system() with tu.ipu_session() as sess: # Clean existing reports sess.run(report) sess.run(variables.global_variables_initializer()) r = sess.run(report) o = sess.run(z) self.assertAllClose(o, 2.0, 0.2, 0.2) s = tu.extract_all_strings_from_event_trace(r) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'vs/z1/Initializer/random_normal/RandomStandardNormal/fusion/normal' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testDepthwiseConvBackpropInput1x1(self): with ops.device("/device:IPU:0"): pa = constant_op.constant([1, 8, 8, 3], dtype=np.int32) # input sizes pb = array_ops.placeholder(np.float32, [1, 1, 3, 2], name="b") pc = array_ops.placeholder(np.float32, [1, 8, 8, 6], name="c") c = nn.depthwise_conv2d_native_backprop_input( pa, pb, pc, strides=[1, 1, 1, 1], padding="SAME") with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = {pb: np.zeros([1, 1, 3, 2]), pc: np.zeros([1, 8, 8, 6])} result = sess.run(c, fd) self.assertAllClose(result, np.zeros([1, 8, 8, 3])) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'DepthwiseConv2dNativeBackpropInput/fusion*/WeightTranspose', 'DepthwiseConv2dNativeBackpropInput/fusion*/Conv_1x1', 'Copy_' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testDefaultTruncatedNormalInitalizer(self): with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() with ops.device("/device:IPU:0"): with variable_scope.variable_scope("", use_resource=True): i = init_ops.truncated_normal_initializer() z = variable_scope.get_variable("z1", shape=[2, 4], dtype=np.float32, initializer=i) tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) o = sess.run(z) self.assertAllClose(o, np.ones((2, 4)), 2.0, 2.0) # Find of the names of compute sets r = sess.run(report) s = tu.extract_all_strings_from_event_trace(r) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'z1/Initializer/truncated_normal/TruncatedNormal/custom-call*/truncatedNormal' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testDontOutlineInplaceExpression(self): with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, []) pb = array_ops.placeholder(np.float32, []) pc = array_ops.placeholder(np.float32, []) pd = array_ops.placeholder(np.float32, []) e = pa + pb - pc + pd with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = {pa: 1, pb: 2, pc: 3, pd: 4} result = sess.run(e, fd) self.assertAllClose(result, 4) result = sess.run(report) self.assertTrue(len(result) == 3) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'add/add.*/AddTo', 'sub/subtract.*/AddTo', 'add_1/add.*/AddTo' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testDepthwiseConvBackpropFilter1x1WithRelu(self): with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, [1, 6, 6, 3], name="a") pb = constant_op.constant([1, 1, 3, 2], dtype=np.int32) # filter sizes pc = array_ops.placeholder(np.float32, [1, 6, 6, 6], name="c") c = nn.depthwise_conv2d_native_backprop_filter( pa, pb, pc, strides=[1, 1, 1, 1], padding="SAME") c = nn.relu(c) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = {pa: np.zeros([1, 6, 6, 3]), pc: np.zeros([1, 6, 6, 6])} result = sess.run(c, fd) self.assertAllClose(result, np.zeros([1, 1, 3, 2])) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'Copy_', 'DepthwiseConv2dNativeBackpropFilter/fusion*/Conv_6x6', 'Relu/custom-call*/Nonlinearity' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testSigmoidGrad(self): with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, [3], name="grad") pb = array_ops.placeholder(np.float32, [3], name="in") c = gen_math_ops.sigmoid_grad(pa, pb) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = {pa: [2.0, 0.5, 1.0], pb: [-1.0, 1.0, 6.0]} result = sess.run(c, fd) self.assertAllClose(result, [2.0, 0.25, 0.0]) result = sess.run(report) self.assertTrue(len(result) == 3) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = ['__seed*', 'SigmoidGrad/custom-call/NonLinearityGrad'] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testNamedOperations(self): with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, [2, 2], name="a") pb = array_ops.placeholder(np.float32, [2, 2], name="b") with ops.name_scope('my_ops'): out = math_ops.add(pa, pb, 'my_add_op') with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: fd = {pa: [[1., 1.], [2., 3.]], pb: [[0., 1.], [4., 5.]]} sess.run(report, fd) result = sess.run(out, fd) self.assertAllClose(result, [[1., 2.], [6., 8.]]) rep = sess.run(report, fd) s = tu.extract_all_strings_from_event_trace(rep) cs_list = tu.get_compute_sets_from_report(s) ok = ['__seed*', 'my_ops/my_add_op/add'] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testUniformRandomNonScalarInitalizer(self): with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() with ops.device("/device:IPU:0"): with variable_scope.variable_scope("vs", use_resource=True): i = init_ops.random_uniform_initializer(minval=-2.0, maxval=2.0) z = variable_scope.get_variable("z1", shape=[2], dtype=np.float32, initializer=i) tu.configure_ipu_system() with tu.ipu_session() as sess: # Clean existing reports sess.run(report) sess.run(variables.global_variables_initializer()) r = sess.run(report) o = sess.run(z) self.assertAllClose(o, [0.0, 0.0], 2.0, 2.0) s = tu.extract_all_strings_from_event_trace(r) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'vs/z1/Initializer/random_uniform/RandomUniform/fusion/uniform' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testScaledSubtractFromVariable(self): with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float16, [3]) pb = array_ops.placeholder(np.float16, [3]) pc = array_ops.placeholder(np.float16, [1]) c = pa - pc * pb with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = {pa: [2.0, 0.5, 1.0], pb: [1.0, 2.0, 3.0], pc: [2.0]} result = sess.run(c, fd) self.assertAllClose(result, [0.0, -3.5, -5.0]) result = sess.run(report) self.assertTrue(len(result) == 3) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = ['__seed*', 'host-exchange-local-copy-', 'sub/fusion/AddTo'] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testScaledSubtractFrom(self): with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float16, [3]) pb = array_ops.placeholder(np.float16, [3]) const = array_ops.constant(2.0, np.float16) # note how const operand index varies compared to testScaledAddTo # still should match as it will be reordered c = pa - const * pb with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = {pa: [2.0, 0.5, 1.0], pb: [1.0, 2.0, 3.0]} result = sess.run(c, fd) self.assertAllClose(result, [0.0, -3.5, -5.0]) result = sess.run(report) self.assertTrue(len(result) == 3) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = ['__seed*', 'host-exchange-local-copy-', 'sub/fusion/AddTo'] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConvolutionBiasApplyVariableLR(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) lr = array_ops.placeholder(np.float32, shape=[]) with variable_scope.variable_scope("vs", use_resource=True): y = layers.Conv2D( 2, 1, use_bias=True, kernel_initializer=init_ops.ones_initializer())(x) y = layers.Conv2D( 2, 1, use_bias=True, kernel_initializer=init_ops.ones_initializer())(y) loss = math_ops.reduce_sum(y) optimizer = gradient_descent.GradientDescentOptimizer(lr) train = optimizer.minimize(loss) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run([train, loss], {x: np.zeros([1, 4, 4, 2]), lr: 0.1}) result = sess.run(report) self.assertEqual( len(result), 6) # 2xcompile, 1xupload, 1xload, 1xdownload, 1xexecute s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'Copy_', 'host-exchange-local-copy-', 'vs/conv2d/BiasAdd/fusion*/addToChannel', 'vs/conv2d/Conv2D/convolution*', 'vs/conv2d_1/BiasAdd/fusion.2/addToChannel', 'GradientDescent/update_vs/conv2d/bias/ResourceApplyGradientDescent/fusion.3/ReduceFinalStage/IntermediateToOutput/Reduce', 'GradientDescent/update_vs/conv2d/bias/ResourceApplyGradientDescent/fusion*/negate/Op/Negate', 'gradients/vs/conv2d_1/Conv2D_grad/Conv2DBackpropFilter/fusion*/Conv_4x4/', 'gradients/vs/conv2d_1/Conv2D_grad/Conv2DBackpropFilter/fusion*/AddTo', 'GradientDescent/update_vs/conv2d_1/bias/ResourceApplyGradientDescent/multiply*/Op/Multiply', 'GradientDescent/update_vs/conv2d_1/bias/ResourceApplyGradientDescent/subtract*/AddTo', 'vs/conv2d/BiasAdd/fusion*/addToChannel', 'Sum/reduce*/ReduceFinalStage/IntermediateToOutput/Reduce', ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testBatchNormAndGroupNormalizeMixedInference(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) with variable_scope.variable_scope("vs", use_resource=True): y = convolutional.conv2d( x, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer()) gamma = constant_op.constant([0.5, 0.5], np.float32) beta = constant_op.constant([0.5, 0.5], np.float32) mean = constant_op.constant([0.5, 0.5], np.float32) inv_std_dev = constant_op.constant([0.5, 0.5], np.float32) y = gen_popnn_ops.popnn_group_norm_inference( inputs=y, gamma=gamma, beta=beta, mean=mean, inv_std_dev=inv_std_dev, data_format="NHWC", epsilon=0.0015, num_groups=2) y = convolutional.conv2d( y, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer()) y = layers_norm.batch_normalization(y, fused=True) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run(y, {x: np.zeros([1, 4, 4, 2])}) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) # Would fail if there were two batch norms in the graph ok = [ '__seed*', 'host-exchange-local-copy', 'Copy_', 'vs/conv2d/Conv2D/convolution.*/Conv_1x1/Convolve', 'vs/PopnnGroupNormInference/custom-call*/', 'vs/batch_normalization/FusedBatchNorm/batch-norm-inference.*/' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConvolutionsMatchFwdBwdWuVariableLR(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) lr = array_ops.placeholder(np.float32, shape=[]) with variable_scope.variable_scope("vs", use_resource=True): y = layers.Conv2D( 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer(), name='conv1')(x) y = layers.Conv2D( 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer(), name='conv2')(y) y = layers.Conv2D( 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer(), name='conv3')(y) loss = math_ops.reduce_sum(y) optimizer = gradient_descent.GradientDescentOptimizer(lr) train = optimizer.minimize(loss) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run([train, loss], {x: np.zeros([1, 4, 4, 2]), lr: 0.1}) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) # Fwd and BackpropInput should be shared # Weight transpose for BackpropInput should be present # Both BackpropFilter should be shared ok = [ '__seed*', 'host-exchange-local-copy-', 'Copy_', 'vs/conv1/Conv2D/convolution.*/Conv_1x1', 'Sum/reduce.*/ReduceFinalStage/IntermediateToOutput/Reduce', 'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*/Conv_4x4', 'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*/AddTo' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testFwdAndBwdMaxPool(self): input = np.arange(16).reshape(1, 4, 4, 1) output_grad = np.full((1, 2, 2, 1), 0.1) with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, [1, 4, 4, 1], name="a") pb = array_ops.placeholder(np.float32, [1, 2, 2, 1], name="b") c = nn.max_pool(pa, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], data_format='NCHW', padding='SAME') d = gen_nn_ops.max_pool_grad(pa, c, pb, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], data_format='NCHW', padding='SAME') with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fe = { pa: input, pb: output_grad, } output, input_grad = sess.run((c, d), fe) self.assertAllClose(output, [[[[5.], [7.]], [[13.], [15.]]]]) self.assertAllClose( input_grad, [[[[0.], [0.], [0.], [0.]], [[0.], [0.1], [0.], [0.1]], [[0.], [0.], [0.], [0.]], [[0.], [0.1], [0.], [0.1]]]]) result = sess.run(report) self.assertTrue(len(result) == 3) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'Copy_*', 'MaxPool/custom-call*/maxPool2x2/', 'MaxPoolGrad/custom-call*/maxPool2x2' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testWideConstantWithAllocationTarget(self): # This test will fail if the dynamic slice is not mapped correctly. dtype = np.float32 shape = (512, 2, 2048) def my_net(y): def cond(i, x, y): return i < 2 def body(i, x, y): s = array_ops.slice(x, [i, i, i], [1, 1, 2048]) y = y + math_ops.reduce_mean(s) i = i + 1 return (i, x, y) i = 0 c = constant_op.constant(4, shape=shape, dtype=dtype, name="c") return control_flow_ops.while_loop(cond, body, (i, c, y))[2] with ops.device('cpu'): y = array_ops.placeholder(dtype, [1]) report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with ops.device("/device:IPU:0"): r = xla.compile(my_net, inputs=[y]) with tu.ipu_session() as sess: sess.run(report) y = sess.run(r, {y: [10]}) self.assertAllClose(y[0], [18]) result = sess.run(report) self.assertTrue(len(result) == 3) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'Copy_*_to_*', 'while/Slice/dynamic-slice*/dynamicSlice', 'while/Mean/reduce', 'while/Mean/multiply', 'while/add*/add*/AddTo' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok)) max_tile_size = tu.get_maximum_tile_size_from_events(s) self.assertTrue(max_tile_size < 60000)
def testBatchNormalizeInferenceDontMatchDifferentTypes(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) with variable_scope.variable_scope("vs", use_resource=True): y = convolutional.conv2d( x, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer()) y = layers_norm.batch_normalization(y, fused=True) y = math_ops.cast(y, np.float16) y = convolutional.conv2d( y, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer()) y = layers_norm.batch_normalization(y, fused=True) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run(y, {x: np.zeros([1, 4, 4, 2])}) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) # Matches two convolutions ok = [ '__seed*', 'host-exchange-local-copy-', 'Copy_', 'vs/conv2d/Conv2D/convolution.*/Conv_1x1', 'vs/batch_normalization/FusedBatchNorm/batch-norm-inference.*/', 'vs/Cast/convert.*/Cast', 'vs/conv2d_1/Conv2D/convolution.*/Conv_1x1', 'vs/batch_normalization_1/FusedBatchNormV2/batch-norm-inference.*/' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testBatchNormalizeInferenceMatchWithSharding(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) with variable_scope.variable_scope("vs", use_resource=True): with tu.ipu_shard(0): a = convolutional.conv2d( x, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer()) b = layers_norm.batch_normalization(a, fused=True) with tu.ipu_shard(0): c = convolutional.conv2d( b, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer()) d = layers_norm.batch_normalization(c, fused=True) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True, sharded=True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run(d, {x: np.zeros([1, 4, 4, 2])}) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) # Would fail if there were two batch norms in the graph ok = [ '__seed*', '*OnTileCopy*', 'vs/conv2d/Conv2D/convolution.*/Conv_1x1/Convolve', 'vs/batch_normalization/FusedBatchNorm/batch-norm-inference.*/' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testAvgPoolSameWithReshape(self): np.random.seed(0) shape = [1, 10, 10, 1] data = np.random.uniform(0, 1, shape) # The expected answer was generated using TF on the cpu expected = [[[[0.64431685], [0.51738459], [0.49705142], [0.60235918], [0.73694557]], [[0.57755166], [0.47387227], [0.40451217], [0.4876942], [0.55843753]], [[0.49037799], [0.4466258], [0.35829377], [0.40070742], [0.37205362]], [[0.47563809], [0.4075647], [0.34894851], [0.35470542], [0.3322109]], [[0.52914065], [0.45464769], [0.38156652], [0.32455513], [0.33199897]]]] with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, shape, name="a") output = nn.avg_pool(pa, ksize=[1, 5, 5, 1], strides=[1, 2, 2, 1], data_format='NHWC', padding='SAME', name="avg") with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) fd = {pa: data} result = sess.run(output, fd) self.assertAllClose(result, expected) result = sess.run(report) self.assertEqual(len(result), 4) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = ['__seed*', 'avg/custom-call*/avgPool5x5'] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConv8x8_WithBias(self): for fmt in self.data_formats: with ops.device("/device:IPU:0"): inp = array_ops.placeholder( np.float32, self._ip_shp([1, 84, 84, 4], fmt), name="inp") wei = array_ops.placeholder(np.float32, [8, 8, 4, 16], name="wei") bia = array_ops.placeholder(np.float32, [16], name="bia") output = nn_ops.conv2d( inp, wei, strides=self._ip_shp([1, 4, 4, 1], fmt), padding="VALID", data_format=fmt, name='cnv4') output = nn_ops.bias_add(output, bia, data_format=fmt, name='ba4') with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = { inp: np.zeros(self._ip_shp([1, 84, 84, 4], fmt)), wei: np.zeros([8, 8, 4, 16]), bia: np.zeros([16]), } result = sess.run(output, fd) self.assertAllClose(result, np.zeros(self._ip_shp([1, 20, 20, 16], fmt))) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'host-exchange-local-copy-', 'Copy_XLA_Args/arg2.*_weights_to_cnv4*/convolution.*/Conv_8x8_stride4x4/weightsRearranged', 'cnv4*/convolution.*/Conv_8x8_stride4x4', 'ba4*/fusion/addToChannel' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def tesInplaceAddCopyWithInplacePeer2(self): data_a = np.array([[10, -10], [-5, 5]]) data_b = np.array([[-15, 15], [25, -25]]) data_c = 2 with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, [2, 2]) pb = array_ops.placeholder(np.float32, [2, 2]) pc = array_ops.placeholder(np.float32, []) a = array_ops.transpose(pa) b = pa + pb * pc c = a * pb + pc d = b / c with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = { pa: data_a, pb: data_b, pc: data_c, } np_result = (data_a + data_b * data_c) / ( np.transpose(data_a) * data_b + data_c) result = sess.run(d, fd) self.assertAllClose(result, np_result) result = sess.run(report) self.assertTrue(len(result) == 3) #compile_begin, compile_end, execute s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'Copy_XLA_Args/arg0.*_to_transpose/transpose' 'mul/multiply.*/Op/Multiply', 'add/add.*/AddTo', 'mul_1/multiply.*/Op/Multiply', 'add_1/add.*/AddTo', 'truediv/divide.*/Op/Divide' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConvolutionsDontMatchDifferentDevices(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) with variable_scope.variable_scope("vs", use_resource=True): with tu.ipu_shard(0): y = layers.Conv2D( 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer())(x) with tu.ipu_shard(1): y = layers.Conv2D( 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer())(y) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True, sharded=True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run(y, {x: np.zeros([1, 4, 4, 2])}) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) # Note how there are two convolutions ok = [ '__seed*', '*OnTileCopy*', 'vs/conv2d/Conv2D/convolution.*', 'Copy_vs/conv2d/Conv2D/convolution.*', 'vs/conv2d_1/Conv2D/convolution.*' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConvolutionsDontMatchDifferentConvParams(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) with variable_scope.variable_scope("vs", use_resource=True): y = layers.Conv2D( 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer())(x) y = layers.Conv2D( 2, 1, use_bias=False, strides=(2, 1), kernel_initializer=init_ops.ones_initializer())(y) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run(y, {x: np.zeros([1, 4, 4, 2])}) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) # Matches two convolutions ok = [ '__seed*', 'Copy_*weightsRearranged', 'host-exchange-local-copy-', 'vs/conv2d/Conv2D/convolution.*/Conv_1x1', 'vs/conv2d_1/Conv2D/convolution.*/Conv_1x1' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def test3DConv8x8x8_WithBias(self): with ops.device("/device:IPU:0"): inp = array_ops.placeholder(np.float32, [1, 84, 84, 84, 2], name="inp") wei = array_ops.placeholder(np.float32, [8, 8, 8, 2, 4], name="wei") bia = array_ops.placeholder(np.float32, [4], name="bia") output = nn_ops.conv3d(inp, wei, strides=[1, 4, 4, 4, 1], padding="VALID") output = nn_ops.bias_add(output, bia) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = { inp: np.zeros([1, 84, 84, 84, 2]), wei: np.zeros([8, 8, 8, 2, 4]), bia: np.zeros([4]), } result = sess.run(output, fd) self.assertAllClose(result, np.zeros([1, 20, 20, 20, 4])) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'host-exchange-local-copy-', 'Copy_', 'Conv3D/convolution.*/Conv_8x8x8_stride4x4x4', 'BiasAdd/fusion/addToChannel' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testAvgPoolValidWithBroadcast(self): np.random.seed(0) shape = [1, 10, 10, 1] data = np.random.uniform(0, 1, shape) # The expected answer was generated using TF on the cpu expected = [[[[0.52647954], [0.44196457], [0.49284577]], [[0.44039682], [0.44067329], [0.44934618]], [[0.46444583], [0.45419583], [0.38236427]]]] with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, shape, name="a") output = nn.avg_pool(pa, ksize=[1, 5, 5, 1], strides=[1, 2, 2, 1], data_format='NHWC', padding='VALID', name="avg") with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) fd = {pa: data} result = sess.run(output, fd) self.assertAllClose(result, expected) result = sess.run(report) self.assertEqual(len(result), 4) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = ['__seed*', 'avg/custom-call*/avgPool5x5'] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConvWithBnAndRelu(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) with variable_scope.variable_scope("vs", use_resource=True): y = layers.Conv2D( 2, 1, use_bias=True, kernel_initializer=init_ops.ones_initializer())(x) y = layers_norm.batch_normalization(y, fused=True) y = nn_ops.relu(y) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run(y, {x: np.zeros([1, 4, 4, 2])}) result = sess.run(report) self.assertEqual( len(result), 6) # 2xcompile, 1xupload 1xload, 1xdownload, 1xexecute s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'host-exchange-local-copy', 'Copy_', 'vs/conv2d/Conv2D/convolution.*/Conv_1x1', 'vs/conv2d/BiasAdd', 'vs/batch_normalization/FusedBatchNorm/batch-norm-inference.*/', 'vs/Relu/custom-call/Nonlinearity' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testInplaceTuple(self): def my_net(x): def cond(i, x, y): return i < 1 def body(i, x, y): i = i + 1 x = nn.tanh(x) y = nn.tanh(y) return (i, x, y) i = 0 return control_flow_ops.while_loop(cond, body, (i, x, x))[1:] with ops.device('cpu'): x = array_ops.placeholder(np.float32, [4]) report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with ops.device("/device:IPU:0"): r = xla.compile(my_net, inputs=[x]) with tu.ipu_session() as sess: sess.run(report) x, y = sess.run(r, {x: np.full([4], 2)}) self.assertAllClose(x, np.full([4], np.tanh(2))) self.assertAllClose(y, np.full([4], np.tanh(2))) result = sess.run(report) self.assertTrue(len(result) == 3) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'Copy_*_to_*', 'while/Tanh/tanh*/Op/Tanh', 'while/Tanh_1/tanh*/Op/Tanh' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConv3x3_WithBias(self): for fmt in self.data_formats: with ops.device("/device:IPU:0"): pa = array_ops.placeholder( np.float32, self._ip_shp([1, 14, 14, 64], fmt), name="a") pb = array_ops.placeholder(np.float32, [3, 3, 64, 128], name="b") bi = array_ops.placeholder(np.float32, [128], name="b") output = nn_ops.convolution( pa, pb, padding="SAME", data_format=fmt, name='cnv3') output = nn_ops.bias_add(output, bi, data_format=fmt, name='ba3') with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = { pa: np.zeros(self._ip_shp([1, 14, 14, 64], fmt)), pb: np.zeros([3, 3, 64, 128]), bi: np.zeros([128]), } result = sess.run(output, fd) self.assertAllClose(result, np.zeros( self._ip_shp([1, 14, 14, 128], fmt))) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'Copy_*actsRearranged', 'host-exchange-local-copy-', 'cnv3*/convolution.*/Conv_3x3', 'ba3*/fusion/addToChannel' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))