def testMultipleOutfeedsRepeatNonTuple(self): outfeed_queue1 = ipu_outfeed_queue.IPUOutfeedQueue(next_feed_id()) outfeed_queue2 = ipu_outfeed_queue.IPUOutfeedQueue(next_feed_id()) def body(v): outfeed1 = outfeed_queue1.enqueue(v) outfeed2 = outfeed_queue2.enqueue(v * 2) v = v + 1 return (v, outfeed1, outfeed2) def my_net(v): r = loops.repeat(20, body, (v)) return r with ops.device('cpu'): v = array_ops.placeholder(np.float32, [4, 4]) with ipu.ops.ipu_scope("/device:IPU:0"): res = ipu_compiler.compile(my_net, inputs=[v]) cfg = ipu.utils.create_ipu_config() cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) ipu.utils.configure_ipu_system(cfg) outfeed1 = outfeed_queue1.dequeue() outfeed2 = outfeed_queue2.dequeue() with session_lib.Session() as sess: with self.assertRaisesRegexp( errors.InvalidArgumentError, 'Only one IPUOutfeedQueue supported per graph'): result = sess.run(res, {v: np.ones([4, 4], np.float32)})
def testTwoOutfeedsDifferentPrograms(self): outfeed_queue1 = ipu_outfeed_queue.IPUOutfeedQueue( feed_name=next_feed_id()) outfeed_queue2 = ipu_outfeed_queue.IPUOutfeedQueue( feed_name=next_feed_id()) def body1(v): outfeed = outfeed_queue1.enqueue(v) v = v + 1 return (v, outfeed) def my_net1(v): r = loops.repeat(5, body1, (v)) return r def body2(v): outfeed = outfeed_queue2.enqueue(v) v = v + 1 return (v, outfeed) def my_net2(v): r = loops.repeat(7, body2, (v)) return r with ops.device('cpu'): v1 = array_ops.placeholder(np.float32, [4, 4]) v2 = array_ops.placeholder(np.float32, [5, 5]) with ipu.ops.ipu_scope("/device:IPU:0"): res1 = ipu_compiler.compile(my_net1, inputs=[v1]) res2 = ipu_compiler.compile(my_net2, inputs=[v2]) cfg = ipu.utils.create_ipu_config() cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) ipu.utils.configure_ipu_system(cfg) outfeed1 = outfeed_queue1.dequeue() outfeed2 = outfeed_queue2.dequeue() with session_lib.Session() as sess: result1 = sess.run(res1, {v1: np.ones([4, 4], np.float32)}) self.assertAllClose(result1[0], np.broadcast_to(6, [4, 4])) outfed1 = sess.run(outfeed1) for i in range(5): self.assertAllClose(outfed1[i], np.broadcast_to(i + 1, [4, 4])) result2 = sess.run(res2, {v2: np.full([5, 5], 4, np.float32)}) self.assertAllClose(result2[0], np.broadcast_to(11, [5, 5])) outfed2 = sess.run(outfeed2) for i in range(7): self.assertAllClose(outfed2[i], np.broadcast_to(i + 4, [5, 5]))
def testSingleInfeedOutfeedRepeatNonTuple(self): dataset = tu.create_single_increasing_dataset(10, shape=[4, 4]) infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id()) outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue(next_feed_id()) def body(v, x): v = v + x outfeed = outfeed_queue.enqueue(v) return (v, outfeed) def my_net(v): r = loops.repeat(20, body, (v), infeed_queue) return r with ops.device('cpu'): v = array_ops.placeholder(np.float32, [4, 4]) with ipu.ops.ipu_scope("/device:IPU:0"): res = ipu_compiler.compile(my_net, inputs=[v]) cfg = ipu.utils.create_ipu_config() cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) ipu.utils.configure_ipu_system(cfg) with session_lib.Session() as sess: sess.run(infeed_queue.initializer) result = sess.run(res, {v: np.ones([4, 4], np.float32)}) self.assertAllClose(result[0], np.broadcast_to(91, [4, 4])) outfed = sess.run(outfeed_queue.dequeue()) self.assertEqual(outfed.shape, (20, 4, 4)) self.assertAllClose(outfed[-1], result[0]) self.assertAllClose(outfed[5], np.broadcast_to(16, [4, 4]))
def testSingleOutfeedRepeatNonTuple(self): outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue(next_feed_id()) def body(v): outfeed = outfeed_queue.enqueue(v) v = v + 1 return (v, outfeed) def my_net(v): r = loops.repeat(20, body, (v)) return r with ops.device('cpu'): v = array_ops.placeholder(np.float32, [4, 4]) with ipu.ops.ipu_scope("/device:IPU:0"): res = ipu_compiler.compile(my_net, inputs=[v]) cfg = ipu.utils.create_ipu_config() cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) ipu.utils.configure_ipu_system(cfg) outfeed = outfeed_queue.dequeue() with session_lib.Session() as sess: result = sess.run(res, {v: np.ones([4, 4], np.float32)}) self.assertAllClose(result[0], np.broadcast_to(21, [4, 4])) outfed = sess.run(outfeed) for i in range(20): self.assertAllClose(outfed[i], np.broadcast_to(i + 1, [4, 4]))
def testCreateSimpleReplicatedOutfeedWrongReplicationFactor(self): shape = [2] dataset = tu.create_single_increasing_dataset(3, shape) outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue( feed_name=next_feed_id(), replication_factor=4) def body(v): v = popops_cross_replica_sum.cross_replica_sum(v) outfeed = outfeed_queue.enqueue(v) return (v, outfeed) def my_net(): v = constant_op.constant(0.0, shape=shape, dtype=np.float32) r = loops.repeat(5, body, [v]) return r with ipu.ops.ipu_scope("/device:IPU:0"): res = ipu_compiler.compile(my_net, inputs=[]) cfg = ipu.utils.create_ipu_config( profiling=False, max_cross_replica_sum_buffer_size=10000) cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) cfg = ipu.utils.auto_select_ipus(cfg, 2) ipu.utils.configure_ipu_system(cfg) with sl.Session() as sess: with self.assertRaisesRegexp( errors.FailedPreconditionError, 'Current program has been created with replication_factor 2' ): result = sess.run(res)
def testErrorWhenNoAllReduce(self): shape = [2] dataset = tu.create_single_increasing_dataset(3, shape) infeed_queue = ipu_infeed_queue.IPUInfeedQueue( dataset, feed_name=next_feed_id(), replication_factor=2) outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue( feed_name=next_feed_id(), replication_factor=2) def body(v, x): outfeed = outfeed_queue.enqueue(v) return (v + x, outfeed) def my_net(): v = constant_op.constant(0.0, shape=shape, dtype=np.float32) r = loops.repeat(5, body, [v], infeed_queue) return r with ipu.ops.ipu_scope("/device:IPU:0"): res = ipu_compiler.compile(my_net, inputs=[]) outfed = outfeed_queue.dequeue() cfg = ipu.utils.create_ipu_config( profiling=False, max_cross_replica_sum_buffer_size=10000) cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) cfg = ipu.utils.auto_select_ipus(cfg, 2) ipu.utils.configure_ipu_system(cfg) with sl.Session() as sess: sess.run(infeed_queue.initializer) with self.assertRaisesRegexp( errors.FailedPreconditionError, 'This is not a valid replicated graph because'): result = sess.run(res)
def testTwoOutfeedsDifferentProgramsSameFeedName(self): outfeed_queue1 = ipu_outfeed_queue.IPUOutfeedQueue(feed_name="a") outfeed_queue2 = ipu_outfeed_queue.IPUOutfeedQueue(feed_name="a") def body1(v): outfeed = outfeed_queue1.enqueue(v) v = v + 1 return (v, outfeed) def my_net1(v): r = loops.repeat(5, body1, (v)) return r def body2(v): outfeed = outfeed_queue2.enqueue(v) v = v + 1 return (v, outfeed) def my_net2(v): r = loops.repeat(7, body2, (v)) return r with ops.device('cpu'): v1 = array_ops.placeholder(np.float32, [4, 4]) v2 = array_ops.placeholder(np.float32, [5, 5]) with ipu.ops.ipu_scope("/device:IPU:0"): res1 = ipu_compiler.compile(my_net1, inputs=[v1]) res2 = ipu_compiler.compile(my_net2, inputs=[v2]) cfg = ipu.utils.create_ipu_config() cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) ipu.utils.configure_ipu_system(cfg) outfeed1 = outfeed_queue1.dequeue() outfeed2 = outfeed_queue2.dequeue() with session_lib.Session() as sess: result1 = sess.run(res1, {v1: np.ones([4, 4], np.float32)}) with self.assertRaisesRegexp( errors.FailedPreconditionError, 'Outfeed with id=\'a\' already exists'): result2 = sess.run(res2, {v2: np.full([5, 5], 4, np.float32)})
def testCreateSimpleReplicatedInfeedOutfeed(self): shape = [2] dataset = tu.create_single_increasing_dataset(3, shape) infeed_queue = ipu_infeed_queue.IPUInfeedQueue( dataset, feed_name=next_feed_id(), replication_factor=2) outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue( feed_name=next_feed_id(), replication_factor=2) def body(v, x): v = popops_cross_replica_sum.cross_replica_sum(v + x) outfeed = outfeed_queue.enqueue(v) return (v, outfeed) def my_net(): v = constant_op.constant(0.0, shape=shape, dtype=np.float32) r = loops.repeat(5, body, [v], infeed_queue) return r with ipu.ops.ipu_scope("/device:IPU:0"): res = ipu_compiler.compile(my_net, inputs=[]) outfed = outfeed_queue.dequeue() cfg = ipu.utils.create_ipu_config( profiling=False, max_cross_replica_sum_buffer_size=10000) cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) cfg = ipu.utils.auto_select_ipus(cfg, 2) ipu.utils.configure_ipu_system(cfg) with sl.Session() as sess: sess.run(infeed_queue.initializer) result = sess.run(res) self.assertAllClose(result[0], np.broadcast_to(48, shape)) outfed_result = sess.run(outfed) self.assertTrue(outfed_result.shape[0], 2) self.assertAllClose(outfed_result[0][0], outfed_result[0][1]) self.assertAllClose(outfed_result[0][0], np.broadcast_to(1, shape)) self.assertAllClose(outfed_result[1][0], outfed_result[1][1]) self.assertAllClose(outfed_result[1][0], np.broadcast_to(4, shape)) self.assertAllClose(outfed_result[2][0], outfed_result[2][1]) self.assertAllClose(outfed_result[2][0], np.broadcast_to(11, shape)) self.assertAllClose(outfed_result[3][0], outfed_result[3][1]) self.assertAllClose(outfed_result[3][0], np.broadcast_to(23, shape)) self.assertAllClose(outfed_result[4][0], outfed_result[4][1]) self.assertAllClose(outfed_result[4][0], np.broadcast_to(48, shape))
def testSingleInfeedOutfeedRepeatNamedLast(self): dataset = tu.create_single_increasing_dataset(3, shape=[4, 4]) shape = [4, 4] def dataset_parser(value): image_1 = value image_2 = (value + 10.) / 2.0 return (image_1, image_2) dataset = dataset.map(dataset_parser) infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id()) outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue( next_feed_id(), outfeed_mode=ipu_outfeed_queue.IPUOutfeedMode.LAST) def body(v, im1, im2): v = v + im1 + im2 outfeed = outfeed_queue.enqueue({ "v": v, "image1": im1, "image2": im2 }) return (v, outfeed) def my_net(): v = constant_op.constant(0.0, shape=shape, dtype=np.float32) r = loops.repeat(5, body, [v], infeed_queue) return r with ipu.ops.ipu_scope("/device:IPU:0"): res = ipu_compiler.compile(my_net, inputs=[]) outfed = outfeed_queue.dequeue() cfg = ipu.utils.create_ipu_config() cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) ipu.utils.configure_ipu_system(cfg) with session_lib.Session() as sess: sess.run(infeed_queue.initializer) result = sess.run(res) self.assertAllClose(result[0], np.broadcast_to(31, shape)) outfed_result = sess.run(outfed) self.assertTrue(len(outfed_result) == 3) self.assertAllClose(outfed_result["v"], np.broadcast_to(31, shape)) self.assertAllClose(outfed_result["image1"], np.broadcast_to(1, shape)) self.assertAllClose(outfed_result["image2"], np.broadcast_to(5.5, shape))
def testTrainingLoopWithInfeedAndOutfeedGetLast(self): dataset = tu.create_single_increasing_dataset(10, shape=[4, 4, 2]) dataset = dataset.batch(batch_size=2, drop_remainder=True) infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id()) outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue( next_feed_id(), outfeed_mode=ipu_outfeed_queue.IPUOutfeedMode.LAST) def my_net(iters): def body(loss, x): with variable_scope.variable_scope("vs", use_resource=True): y = layers.Conv2D( 2, 1, use_bias=True, kernel_initializer=init_ops.ones_initializer(), name='conv1')(x) loss = math_ops.reduce_sum(y) optimizer = gradient_descent.GradientDescentOptimizer(0.1) train = optimizer.minimize(loss) outfeed = outfeed_queue.enqueue(loss) with ops.control_dependencies([train]): return (array_ops.identity(loss), outfeed) loss = 0.0 return loops.repeat(iters, body, (loss), infeed_queue) with ops.device('cpu'): iters = array_ops.placeholder(np.int32, shape=[]) with ipu.ops.ipu_scope("/device:IPU:0"): r = ipu_compiler.compile(my_net, inputs=[iters]) outfeeds = outfeed_queue.dequeue() with session_lib.Session() as sess: sess.run(infeed_queue.initializer) sess.run(variables.global_variables_initializer()) initial_loss = sess.run(r, {iters: 1}) final_loss = sess.run(r, {iters: 1000}) outfed = sess.run(outfeeds) self.assertTrue(initial_loss > final_loss) self.assertTrue(outfed == final_loss) # Check that a scalar is returned instead of a numpy array self.assertTrue(type(outfed) == np.float32)