def testConvolutionsMatchFwdBwdWuVariableLR(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) lr = array_ops.placeholder(np.float32, shape=[]) with variable_scope.variable_scope("vs", use_resource=True): y = layers.Conv2D( 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer(), name='conv1')(x) y = layers.Conv2D( 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer(), name='conv2')(y) y = layers.Conv2D( 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer(), name='conv3')(y) loss = math_ops.reduce_sum(y) optimizer = gradient_descent.GradientDescentOptimizer(lr) train = optimizer.minimize(loss) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run([train, loss], {x: np.zeros([1, 4, 4, 2]), lr: 0.1}) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) # Fwd and BackpropInput should be shared # Weight transpose for BackpropInput should be present # Both BackpropFilter should be shared ok = [ '__seed*', 'host-exchange-local-copy-', 'Copy_', 'vs/conv1/Conv2D/convolution.*/Conv_1x1', 'Sum/reduce.*/ReduceFinalStage/IntermediateToOutput/Reduce', 'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*/Conv_4x4', 'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*/AddTo' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testBatchNormAndGroupNormalizeMixedInference(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) with variable_scope.variable_scope("vs", use_resource=True): y = convolutional.conv2d( x, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer()) gamma = constant_op.constant([0.5, 0.5], np.float32) beta = constant_op.constant([0.5, 0.5], np.float32) mean = constant_op.constant([0.5, 0.5], np.float32) inv_std_dev = constant_op.constant([0.5, 0.5], np.float32) y = gen_popnn_ops.popnn_group_norm_inference( inputs=y, gamma=gamma, beta=beta, mean=mean, inv_std_dev=inv_std_dev, data_format="NHWC", epsilon=0.0015, num_groups=2) y = convolutional.conv2d( y, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer()) y = layers_norm.batch_normalization(y, fused=True) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run(y, {x: np.zeros([1, 4, 4, 2])}) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) # Would fail if there were two batch norms in the graph ok = [ '__seed*', 'host-exchange-local-copy', 'Copy_', 'vs/conv2d/Conv2D/convolution.*/Conv_1x1/Convolve', 'vs/PopnnGroupNormInference/custom-call*/', 'vs/batch_normalization/FusedBatchNorm/batch-norm-inference.*/' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testFwdAndBwdMaxPool(self): input = np.arange(16).reshape(1, 4, 4, 1) output_grad = np.full((1, 2, 2, 1), 0.1) with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, [1, 4, 4, 1], name="a") pb = array_ops.placeholder(np.float32, [1, 2, 2, 1], name="b") c = nn.max_pool(pa, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], data_format='NCHW', padding='SAME') d = gen_nn_ops.max_pool_grad(pa, c, pb, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], data_format='NCHW', padding='SAME') with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fe = { pa: input, pb: output_grad, } output, input_grad = sess.run((c, d), fe) self.assertAllClose(output, [[[[5.], [7.]], [[13.], [15.]]]]) self.assertAllClose( input_grad, [[[[0.], [0.], [0.], [0.]], [[0.], [0.1], [0.], [0.1]], [[0.], [0.], [0.], [0.]], [[0.], [0.1], [0.], [0.1]]]]) result = sess.run(report) self.assertTrue(len(result) == 3) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'Copy_*', 'MaxPool/custom-call*/maxPool2x2/', 'MaxPoolGrad/custom-call*/maxPool2x2' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testWideConstantWithAllocationTarget(self): # This test will fail if the dynamic slice is not mapped correctly. dtype = np.float32 shape = (512, 2, 2048) def my_net(y): def cond(i, x, y): return i < 2 def body(i, x, y): s = array_ops.slice(x, [i, i, i], [1, 1, 2048]) y = y + math_ops.reduce_mean(s) i = i + 1 return (i, x, y) i = 0 c = constant_op.constant(4, shape=shape, dtype=dtype, name="c") return control_flow_ops.while_loop(cond, body, (i, c, y))[2] with ops.device('cpu'): y = array_ops.placeholder(dtype, [1]) report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with ops.device("/device:IPU:0"): r = xla.compile(my_net, inputs=[y]) with tu.ipu_session() as sess: sess.run(report) y = sess.run(r, {y: [10]}) self.assertAllClose(y[0], [18]) result = sess.run(report) self.assertTrue(len(result) == 3) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'Copy_*_to_*', 'while/Slice/dynamic-slice*/dynamicSlice', 'while/Mean/reduce', 'while/Mean/multiply', 'while/add*/add*/AddTo' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok)) max_tile_size = tu.get_maximum_tile_size_from_events(s) self.assertTrue(max_tile_size < 60000)
def testBatchNormalizeInferenceMatchWithSharding(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) with variable_scope.variable_scope("vs", use_resource=True): with tu.ipu_shard(0): a = convolutional.conv2d( x, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer()) b = layers_norm.batch_normalization(a, fused=True) with tu.ipu_shard(0): c = convolutional.conv2d( b, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer()) d = layers_norm.batch_normalization(c, fused=True) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True, sharded=True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run(d, {x: np.zeros([1, 4, 4, 2])}) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) # Would fail if there were two batch norms in the graph ok = [ '__seed*', '*OnTileCopy*', 'vs/conv2d/Conv2D/convolution.*/Conv_1x1/Convolve', 'vs/batch_normalization/FusedBatchNorm/batch-norm-inference.*/' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testBatchNormalizeInferenceDontMatchDifferentTypes(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) with variable_scope.variable_scope("vs", use_resource=True): y = convolutional.conv2d( x, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer()) y = layers_norm.batch_normalization(y, fused=True) y = math_ops.cast(y, np.float16) y = convolutional.conv2d( y, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer()) y = layers_norm.batch_normalization(y, fused=True) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run(y, {x: np.zeros([1, 4, 4, 2])}) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) # Matches two convolutions ok = [ '__seed*', 'host-exchange-local-copy-', 'Copy_', 'vs/conv2d/Conv2D/convolution.*/Conv_1x1', 'vs/batch_normalization/FusedBatchNorm/batch-norm-inference.*/', 'vs/Cast/convert.*/Cast', 'vs/conv2d_1/Conv2D/convolution.*/Conv_1x1', 'vs/batch_normalization_1/FusedBatchNormV2/batch-norm-inference.*/' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testAvgPoolSameWithReshape(self): np.random.seed(0) shape = [1, 10, 10, 1] data = np.random.uniform(0, 1, shape) # The expected answer was generated using TF on the cpu expected = [[[[0.64431685], [0.51738459], [0.49705142], [0.60235918], [0.73694557]], [[0.57755166], [0.47387227], [0.40451217], [0.4876942], [0.55843753]], [[0.49037799], [0.4466258], [0.35829377], [0.40070742], [0.37205362]], [[0.47563809], [0.4075647], [0.34894851], [0.35470542], [0.3322109]], [[0.52914065], [0.45464769], [0.38156652], [0.32455513], [0.33199897]]]] with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, shape, name="a") output = nn.avg_pool(pa, ksize=[1, 5, 5, 1], strides=[1, 2, 2, 1], data_format='NHWC', padding='SAME', name="avg") with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) fd = {pa: data} result = sess.run(output, fd) self.assertAllClose(result, expected) result = sess.run(report) self.assertEqual(len(result), 4) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = ['__seed*', 'avg/custom-call*/avgPool5x5'] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConvolutionBiasApply(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) with variable_scope.variable_scope("vs", use_resource=True): y = layers.Conv2D( 2, 1, use_bias=True, kernel_initializer=init_ops.ones_initializer())(x) y = layers.Conv2D( 2, 1, use_bias=True, kernel_initializer=init_ops.ones_initializer())(y) loss = math_ops.reduce_sum(y) optimizer = gradient_descent.GradientDescentOptimizer(0.1) train = optimizer.minimize(loss) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run([train, loss], {x: np.zeros([1, 4, 4, 2])}) result = sess.run(report) self.assertEqual( len(result), 6) # 2xcompile, 1xupload, 1xload, 1xdownload, 1xexecute s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'GradientDescent/update_vs/conv2d/bias/ResourceApplyGradientDescent/fusion.*/Reduce' ] self.assertTrue( tu.check_compute_sets_in_whitelist_entries(cs_list, ok))
def testBatchNormalizeFused(self): x = array_ops.placeholder(np.float32, [4, 64, 64, 4], name="a") with ops.device("/device:IPU:0"): with variable_scope.variable_scope("", use_resource=True): beta = variable_scope.get_variable( "x", dtype=np.float32, shape=[4], initializer=init_ops.constant_initializer(0.0)) gamma = variable_scope.get_variable( "y", dtype=np.float32, shape=[4], initializer=init_ops.constant_initializer(1.0)) b_mean, b_var = nn.moments(x, [0, 1, 2], name='moments') normed = nn.fused_batch_norm(x, gamma, beta, b_mean, b_var, is_training=False) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) sess.run(variables.global_variables_initializer()) result, _, _ = sess.run(normed, {x: np.zeros([4, 64, 64, 4])}) self.assertAllClose(result, np.zeros([4, 64, 64, 4])) rep = sess.run(report) s = tu.extract_all_strings_from_event_trace(rep) cs = tu.get_compute_sets_from_report(s) bl = ['*convert*/Cast*'] self.assertTrue(tu.check_compute_sets_not_in_blacklist(cs, bl))
def testConv8x8_WithBias(self): for fmt in self.data_formats: with ops.device("/device:IPU:0"): inp = array_ops.placeholder( np.float32, self._ip_shp([1, 84, 84, 4], fmt), name="inp") wei = array_ops.placeholder(np.float32, [8, 8, 4, 16], name="wei") bia = array_ops.placeholder(np.float32, [16], name="bia") output = nn_ops.conv2d( inp, wei, strides=self._ip_shp([1, 4, 4, 1], fmt), padding="VALID", data_format=fmt, name='cnv4') output = nn_ops.bias_add(output, bia, data_format=fmt, name='ba4') with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = { inp: np.zeros(self._ip_shp([1, 84, 84, 4], fmt)), wei: np.zeros([8, 8, 4, 16]), bia: np.zeros([16]), } result = sess.run(output, fd) self.assertAllClose(result, np.zeros(self._ip_shp([1, 20, 20, 16], fmt))) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'host-exchange-local-copy-', 'Copy_XLA_Args/arg2.*_weights_to_cnv4*/convolution.*/Conv_8x8_stride4x4/weightsRearranged', 'cnv4*/convolution.*/Conv_8x8_stride4x4', 'ba4*/fusion/addToChannel' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def tesInplaceAddCopyWithInplacePeer2(self): data_a = np.array([[10, -10], [-5, 5]]) data_b = np.array([[-15, 15], [25, -25]]) data_c = 2 with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, [2, 2]) pb = array_ops.placeholder(np.float32, [2, 2]) pc = array_ops.placeholder(np.float32, []) a = array_ops.transpose(pa) b = pa + pb * pc c = a * pb + pc d = b / c with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = { pa: data_a, pb: data_b, pc: data_c, } np_result = (data_a + data_b * data_c) / ( np.transpose(data_a) * data_b + data_c) result = sess.run(d, fd) self.assertAllClose(result, np_result) result = sess.run(report) self.assertTrue(len(result) == 3) #compile_begin, compile_end, execute s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'Copy_XLA_Args/arg0.*_to_transpose/transpose' 'mul/multiply.*/Op/Multiply', 'add/add.*/AddTo', 'mul_1/multiply.*/Op/Multiply', 'add_1/add.*/AddTo', 'truediv/divide.*/Op/Divide' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConvolutionsDontMatchDifferentDevices(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) with variable_scope.variable_scope("vs", use_resource=True): with tu.ipu_shard(0): y = layers.Conv2D( 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer())(x) with tu.ipu_shard(1): y = layers.Conv2D( 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer())(y) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True, sharded=True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run(y, {x: np.zeros([1, 4, 4, 2])}) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) # Note how there are two convolutions ok = [ '__seed*', '*OnTileCopy*', 'vs/conv2d/Conv2D/convolution.*', 'Copy_vs/conv2d/Conv2D/convolution.*', 'vs/conv2d_1/Conv2D/convolution.*' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConvolutionsDontMatchDifferentConvParams(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) with variable_scope.variable_scope("vs", use_resource=True): y = layers.Conv2D( 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer())(x) y = layers.Conv2D( 2, 1, use_bias=False, strides=(2, 1), kernel_initializer=init_ops.ones_initializer())(y) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run(y, {x: np.zeros([1, 4, 4, 2])}) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) # Matches two convolutions ok = [ '__seed*', 'Copy_*weightsRearranged', 'host-exchange-local-copy-', 'vs/conv2d/Conv2D/convolution.*/Conv_1x1', 'vs/conv2d_1/Conv2D/convolution.*/Conv_1x1' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConvWithBnAndRelu(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) with variable_scope.variable_scope("vs", use_resource=True): y = layers.Conv2D( 2, 1, use_bias=True, kernel_initializer=init_ops.ones_initializer())(x) y = layers_norm.batch_normalization(y, fused=True) y = nn_ops.relu(y) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run(y, {x: np.zeros([1, 4, 4, 2])}) result = sess.run(report) self.assertEqual( len(result), 6) # 2xcompile, 1xupload 1xload, 1xdownload, 1xexecute s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'host-exchange-local-copy', 'Copy_', 'vs/conv2d/Conv2D/convolution.*/Conv_1x1', 'vs/conv2d/BiasAdd', 'vs/batch_normalization/FusedBatchNorm/batch-norm-inference.*/', 'vs/Relu/custom-call/Nonlinearity' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testAvgPoolValidWithBroadcast(self): np.random.seed(0) shape = [1, 10, 10, 1] data = np.random.uniform(0, 1, shape) # The expected answer was generated using TF on the cpu expected = [[[[0.52647954], [0.44196457], [0.49284577]], [[0.44039682], [0.44067329], [0.44934618]], [[0.46444583], [0.45419583], [0.38236427]]]] with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, shape, name="a") output = nn.avg_pool(pa, ksize=[1, 5, 5, 1], strides=[1, 2, 2, 1], data_format='NHWC', padding='VALID', name="avg") with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) fd = {pa: data} result = sess.run(output, fd) self.assertAllClose(result, expected) result = sess.run(report) self.assertEqual(len(result), 4) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = ['__seed*', 'avg/custom-call*/avgPool5x5'] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def test3DConv8x8x8_WithBias(self): with ops.device("/device:IPU:0"): inp = array_ops.placeholder(np.float32, [1, 84, 84, 84, 2], name="inp") wei = array_ops.placeholder(np.float32, [8, 8, 8, 2, 4], name="wei") bia = array_ops.placeholder(np.float32, [4], name="bia") output = nn_ops.conv3d(inp, wei, strides=[1, 4, 4, 4, 1], padding="VALID") output = nn_ops.bias_add(output, bia) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = { inp: np.zeros([1, 84, 84, 84, 2]), wei: np.zeros([8, 8, 8, 2, 4]), bia: np.zeros([4]), } result = sess.run(output, fd) self.assertAllClose(result, np.zeros([1, 20, 20, 20, 4])) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'host-exchange-local-copy-', 'Copy_', 'Conv3D/convolution.*/Conv_8x8x8_stride4x4x4', 'BiasAdd/fusion/addToChannel' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testInplaceTuple(self): def my_net(x): def cond(i, x, y): return i < 1 def body(i, x, y): i = i + 1 x = nn.tanh(x) y = nn.tanh(y) return (i, x, y) i = 0 return control_flow_ops.while_loop(cond, body, (i, x, x))[1:] with ops.device('cpu'): x = array_ops.placeholder(np.float32, [4]) report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with ops.device("/device:IPU:0"): r = xla.compile(my_net, inputs=[x]) with tu.ipu_session() as sess: sess.run(report) x, y = sess.run(r, {x: np.full([4], 2)}) self.assertAllClose(x, np.full([4], np.tanh(2))) self.assertAllClose(y, np.full([4], np.tanh(2))) result = sess.run(report) self.assertTrue(len(result) == 3) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'Copy_*_to_*', 'while/Tanh/tanh*/Op/Tanh', 'while/Tanh_1/tanh*/Op/Tanh' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testConv3x3_WithBias(self): for fmt in self.data_formats: with ops.device("/device:IPU:0"): pa = array_ops.placeholder( np.float32, self._ip_shp([1, 14, 14, 64], fmt), name="a") pb = array_ops.placeholder(np.float32, [3, 3, 64, 128], name="b") bi = array_ops.placeholder(np.float32, [128], name="b") output = nn_ops.convolution( pa, pb, padding="SAME", data_format=fmt, name='cnv3') output = nn_ops.bias_add(output, bi, data_format=fmt, name='ba3') with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = { pa: np.zeros(self._ip_shp([1, 14, 14, 64], fmt)), pb: np.zeros([3, 3, 64, 128]), bi: np.zeros([128]), } result = sess.run(output, fd) self.assertAllClose(result, np.zeros( self._ip_shp([1, 14, 14, 128], fmt))) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'Copy_*actsRearranged', 'host-exchange-local-copy-', 'cnv3*/convolution.*/Conv_3x3', 'ba3*/fusion/addToChannel' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def test3DConvBackpropInput(self): with ops.device("/device:IPU:0"): ins = constant_op.constant([2, 8, 8, 8, 3], np.int32) fil = array_ops.placeholder(np.float32, [2, 2, 2, 3, 5], name="inp") bck = array_ops.placeholder(np.float32, [2, 8, 8, 8, 5], name="wei") output = nn_ops.conv3d_backprop_input_v2(ins, fil, bck, strides=[1, 1, 1, 1, 1], padding="SAME") with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = { fil: np.zeros([2, 2, 2, 3, 5]), bck: np.zeros([2, 8, 8, 8, 5]), } result = sess.run(output, fd) self.assertAllClose(result, np.zeros([2, 8, 8, 8, 3])) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'Copy_', 'Conv3DBackpropInputV2/fusion*/Conv_2x2x2' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testFullyConnectedWithBias(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[2, 2]) weights = array_ops.placeholder(np.float32, shape=[2, 2]) bias = array_ops.placeholder(np.float32, shape=[2]) x_new = nn.xw_plus_b(x, weights, bias) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(report) out = sess.run( x_new, { x: np.full([2, 2], 3), weights: np.full([2, 2], 4), bias: np.ones([2]), }) self.assertAllClose(np.full([2, 2], 25), out) result = sess.run(report) self.assertEqual(len(result), 4) # 1xcompile, 1xload, 1xdownload, 1xexecute s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'host-exchange-local-copy', 'xw_plus_b/MatMul/dot.*/Conv_1/Convolve', 'xw_plus_b/fusion/addToChannel' ] self.assertTrue( tu.check_compute_sets_in_whitelist_entries(cs_list, ok))
def testInplaceOpAddCopyWithInplaceParent(self): with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, [3]) pb = array_ops.placeholder(np.float32, [3]) pc = array_ops.placeholder(np.float32, []) c = array_ops.slice(pa, [0], [2]) d = array_ops.slice(pb, [0], [2]) e = c + d f = e / pc g = array_ops.slice(pa, [1], [2]) h = f + g with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = { pa: [1, 2, 3], pb: [5, 6, 7], pc: 2, } result = sess.run(h, fd) self.assertAllClose(result, [5, 7]) result = sess.run(report) self.assertTrue(len(result) == 3) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'Copy_XLA_Args/arg*_to_Slice*/slice*.clone', 'add/add.*/AddTo', 'truediv/divide.*/Op/Divide', 'add_1/add.*/AddTo' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testRelu(self): with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, [3], name="a") c = nn_ops.relu(pa) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: fd = {pa: [-6.0, 0.0, 6.0]} result = sess.run(c, fd) self.assertAllClose(result, [0.0, 0.0, 6.0]) result = sess.run(report) self.assertTrue(len(result) == 3) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = ['__seed*', 'Relu/custom-call/Nonlinearity'] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testDepthwiseConv3x1(self): with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, [1, 2, 2, 3], name="a") pb = array_ops.placeholder(np.float32, [1, 1, 3, 1], name="b") pc = array_ops.placeholder(np.float32, [3], name="c") c = nn.depthwise_conv2d(pa, pb, strides=[1, 1, 1, 1], padding="SAME") output = c + pc with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = { pa: [[[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]]], pb: [[[[6], [4], [2]]]], pc: [1, 1, 1] } result = sess.run(output, fd) self.assertAllClose( result, [[[[7, 9, 7], [25, 21, 13]], [[43, 33, 19], [61, 45, 25]]]]) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'host-exchange-local-copy-', 'Copy_', 'depthwise/convolution.*/Conv_1x1', 'Copy_depthwise/convolution.*/Conv_1x1/partials_to_depthwise/convolution.*/Conv_1x1/partials[[]cloned[]]', 'add/fusion*/addToChannel' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testTruncatedNormalInitalizer(self): with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() with ops.device("/device:IPU:0"): with variable_scope.variable_scope("", use_resource=True): i = init_ops.truncated_normal_initializer(mean=1.0, stddev=0.01) z = variable_scope.get_variable("z1", shape=[2, 4], dtype=np.float32, initializer=i) tu.configure_ipu_system() with tu.ipu_session() as sess: # Clean existing reports sess.run(report) sess.run(variables.global_variables_initializer()) o = sess.run(z) self.assertAllClose(o, np.ones((2, 4)), 0.2, 0.2) # Find of the names of compute sets r = sess.run(report) s = tu.extract_all_strings_from_event_trace(r) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'z1/Initializer/truncated_normal/TruncatedNormal/custom-call*/truncatedNormal', 'z1/Initializer/truncated_normal/mul/multiply.*/Op/Multiply', 'z1/Initializer/truncated_normal/add.*/AddTo' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def tesInplaceAddCopyWithInplacePeer(self): data_a = np.array([[10, -20], [5, 1]]) data_b = np.array([[-12, 11], [12, -13]]) with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, [2, 2]) pb = array_ops.placeholder(np.float32, [2, 2]) c = array_ops.transpose(pa) d = pa + pb e = c / d with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = { pa: data_a, pb: data_b, } result = sess.run(e, fd) np_result = np.transpose(data_a) / (data_a + data_b) self.assertAllClose(result, np_result) result = sess.run(report) self.assertTrue(len(result) == 3) #compile_begin, compile_end, execute s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'host-exchange-local-copy-', 'Copy_XLA_Args/arg0.*_to_transpose/transpose', 'add/add.*/AddTo', 'truediv/divide.*/Op/Divide' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def test3DConv3x3x3_WithBias(self): with ops.device("/device:IPU:0"): pa = array_ops.placeholder(np.float32, [1, 14, 14, 14, 16], name="a") pb = array_ops.placeholder(np.float32, [3, 3, 3, 16, 32], name="b") bi = array_ops.placeholder(np.float32, [32], name="b") output = nn_ops.convolution(pa, pb, padding="SAME") output = nn_ops.bias_add(output, bi) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = { pa: np.zeros([1, 14, 14, 14, 16]), pb: np.zeros([3, 3, 3, 16, 32]), bi: np.zeros([32]), } result = sess.run(output, fd) self.assertAllClose(result, np.zeros([1, 14, 14, 14, 32])) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = [ '__seed*', 'host-exchange-local-copy-', 'Copy_', 'convolution/convolution.*/Conv_3x3x3', 'BiasAdd/fusion/addToChannel' ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testBatchNormsMatchFwdBwd(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) with variable_scope.variable_scope("vs", use_resource=True): y = convolutional.conv2d( x, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer(), name='conv1') y = layers_norm.batch_normalization(y, fused=True, training=True) y = convolutional.conv2d( y, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer(), name='conv2') y = layers_norm.batch_normalization(y, fused=True, training=True) y = convolutional.conv2d( y, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer(), name='conv3') y = layers_norm.batch_normalization(y, fused=True, training=True) loss = math_ops.reduce_sum(y) optimizer = gradient_descent.GradientDescentOptimizer(0.1) train = optimizer.minimize(loss) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run([train, loss], {x: np.zeros([1, 4, 4, 2])}) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) # One BN for forwards and one BN for grad # (note that we don't cache gradient application) ok = [ '__seed*', 'host-exchange-local-copy-', 'Copy_', 'vs/conv1/Conv2D/convolution.*/Conv_1x1', 'vs/batch_normalization/FusedBatchNorm/batch-norm-training.*/', 'Sum/reduce.*/ReduceFinalStage/IntermediateToOutput/Reduce', 'gradients/vs/batch_normalization_2/FusedBatchNorm_grad/FusedBatchNormGrad/batch-norm-grad.*/', 'GradientDescent/update_vs/batch_normalization/', 'GradientDescent/update_vs/batch_normalization_1/', 'GradientDescent/update_vs/batch_normalization_2/', 'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*/Conv_4x4/Convolve', 'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*/AddTo', ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def testGroupNormsMatchFwdBwd(self): with ops.device("/device:IPU:0"): x = array_ops.placeholder(np.float32, shape=[1, 4, 4, 2]) with variable_scope.variable_scope("vs", use_resource=True): y = convolutional.conv2d( x, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer(), name='conv1') gamma = constant_op.constant([0.5, 0.5], np.float32) beta = constant_op.constant([0.5, 0.5], np.float32) y, _, _ = gen_popnn_ops.popnn_group_norm_training( inputs=y, gamma=gamma, beta=beta, data_format="NHWC", epsilon=0.0015, num_groups=2) y = convolutional.conv2d( y, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer(), name='conv2') y, _, _ = gen_popnn_ops.popnn_group_norm_training( inputs=y, gamma=gamma, beta=beta, data_format="NHWC", epsilon=0.0015, num_groups=2) y = convolutional.conv2d( y, 2, 1, use_bias=False, kernel_initializer=init_ops.ones_initializer(), name='conv3') y, _, _ = gen_popnn_ops.popnn_group_norm_training( inputs=y, gamma=gamma, beta=beta, data_format="NHWC", epsilon=0.0015, num_groups=2) loss = math_ops.reduce_sum(y) optimizer = gradient_descent.GradientDescentOptimizer(0.1) train = optimizer.minimize(loss) with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system(True, True, True) with tu.ipu_session() as sess: sess.run(variables.global_variables_initializer()) sess.run(report) sess.run([train, loss], {x: np.zeros([1, 4, 4, 2])}) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) # One GN for forwards and one GN for grad ok = [ '__seed*', 'host-exchange-local-copy-', 'Copy_', 'vs/conv1/Conv2D/convolution*/Conv_1x1/Convolve', 'vs/PopnnGroupNormTraining/custom-call*/Norm', 'vs/PopnnGroupNormTraining/custom-call*/iStdDev', 'vs/PopnnGroupNormTraining/custom-call*/Whiten', 'Sum/reduce.*/*/Reduce', 'gradients/vs/PopnnGroupNormTraining_2_grad/PopnnGroupNormGrad/custom-call*/', 'gradients/vs/conv3/Conv2D_grad/Conv2DBackpropFilter/fusion.*', ] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))