def testSingleInfeedMultipleRuns(self): dataset = tu.create_single_increasing_dataset(10, shape=[4, 4]) infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id()) def program(iters): def body(v, x): v = v + x return (v) def my_net(): v = constant_op.constant(0.0, shape=[4, 4], dtype=np.float32) r = loops.repeat(iters, body, (v), infeed_queue) return r with ipu.ops.ipu_scope("/device:IPU:0"): return ipu_compiler.compile(my_net) cfg = ipu.utils.create_ipu_config() cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) ipu.utils.configure_ipu_system(cfg) with session_lib.Session() as sess: sess.run(infeed_queue.initializer) result = sess.run(program(0)) self.assertAllClose(result[0], np.broadcast_to(0, [4, 4])) # The iterator has not moved - next element should be all 1s. result = sess.run(program(2)) self.assertAllClose(result[0], np.broadcast_to(1, [4, 4])) # The iterator has moved - in the next two iterations it should pull 2 and 3. result = sess.run(program(2)) self.assertAllClose(result[0], np.broadcast_to(5, [4, 4])) # The iterator has moved - in the next two iterations it should pull 4 and 5. result = sess.run(program(2)) self.assertAllClose(result[0], np.broadcast_to(9, [4, 4]))
def testSingleInfeedRepeatNonTupleFiniteDataset(self): dataset = tu.create_single_increasing_dataset(10, shape=[4, 4], repeat=False) infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id()) def body(v, x): v = v + x return (v) def my_net(v): r = loops.repeat(10, body, (v), infeed_queue) return r with ops.device('cpu'): v = array_ops.placeholder(np.float32, [4, 4]) with ipu.ops.ipu_scope("/device:IPU:0"): res = ipu_compiler.compile(my_net, inputs=[v]) cfg = ipu.utils.create_ipu_config() cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) ipu.utils.configure_ipu_system(cfg) with session_lib.Session() as sess: sess.run(infeed_queue.initializer) result = sess.run(res, {v: np.ones([4, 4], np.float32)}) self.assertAllClose(result[0], np.broadcast_to(46, [4, 4]))
def testSingleInfeedMultipleRepeats(self): dataset = tu.create_single_increasing_dataset(2, shape=[4, 4]) infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id()) def body(v, x): v = v + x return (v) def my_net(): v = constant_op.constant(0.0, shape=[4, 4], dtype=np.float32) r = loops.repeat(5, body, [v], infeed_queue) r = loops.repeat(5, body, [r], infeed_queue) return r with ipu.ops.ipu_scope("/device:IPU:0"): res = ipu_compiler.compile(my_net, inputs=[]) cfg = ipu.utils.create_ipu_config() cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) ipu.utils.configure_ipu_system(cfg) with session_lib.Session() as sess: sess.run(infeed_queue.initializer) result = sess.run(res) self.assertAllClose(result[0], np.broadcast_to(5, [4, 4]))
def testSingleInfeedRepeatTupleMerge(self): dataset = tu.create_single_increasing_dataset(3, shape=[4, 4]) def dataset_parser(value): image_1 = value image_2 = (value + 10.) / 2.0 return (image_1, image_2) dataset = dataset.map(dataset_parser) infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id()) def body(v, im1, im2): v = v + im1 + im2 return (v) def my_net(): v = constant_op.constant(0.0, shape=[4, 4], dtype=np.float32) r = loops.repeat(5, body, [v], infeed_queue) return r with ipu.ops.ipu_scope("/device:IPU:0"): res = ipu_compiler.compile(my_net, inputs=[]) cfg = ipu.utils.create_ipu_config(merge_infeed_io_copies=True) cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) ipu.utils.configure_ipu_system(cfg) with session_lib.Session() as sess: sess.run(infeed_queue.initializer) result = sess.run(res) self.assertAllClose(result[0], np.broadcast_to(31, [4, 4]))
def testCreateSimpleReplicatedOutfeedWrongReplicationFactor(self): shape = [2] dataset = tu.create_single_increasing_dataset(3, shape) outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue( feed_name=next_feed_id(), replication_factor=4) def body(v): v = popops_cross_replica_sum.cross_replica_sum(v) outfeed = outfeed_queue.enqueue(v) return (v, outfeed) def my_net(): v = constant_op.constant(0.0, shape=shape, dtype=np.float32) r = loops.repeat(5, body, [v]) return r with ipu.ops.ipu_scope("/device:IPU:0"): res = ipu_compiler.compile(my_net, inputs=[]) cfg = ipu.utils.create_ipu_config( profiling=False, max_cross_replica_sum_buffer_size=10000) cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) cfg = ipu.utils.auto_select_ipus(cfg, 2) ipu.utils.configure_ipu_system(cfg) with sl.Session() as sess: with self.assertRaisesRegexp( errors.FailedPreconditionError, 'Current program has been created with replication_factor 2' ): result = sess.run(res)
def testErrorWhenNoAllReduce(self): shape = [2] dataset = tu.create_single_increasing_dataset(3, shape) infeed_queue = ipu_infeed_queue.IPUInfeedQueue( dataset, feed_name=next_feed_id(), replication_factor=2) outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue( feed_name=next_feed_id(), replication_factor=2) def body(v, x): outfeed = outfeed_queue.enqueue(v) return (v + x, outfeed) def my_net(): v = constant_op.constant(0.0, shape=shape, dtype=np.float32) r = loops.repeat(5, body, [v], infeed_queue) return r with ipu.ops.ipu_scope("/device:IPU:0"): res = ipu_compiler.compile(my_net, inputs=[]) outfed = outfeed_queue.dequeue() cfg = ipu.utils.create_ipu_config( profiling=False, max_cross_replica_sum_buffer_size=10000) cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) cfg = ipu.utils.auto_select_ipus(cfg, 2) ipu.utils.configure_ipu_system(cfg) with sl.Session() as sess: sess.run(infeed_queue.initializer) with self.assertRaisesRegexp( errors.FailedPreconditionError, 'This is not a valid replicated graph because'): result = sess.run(res)
def testMultipleInitializations(self): dataset = tu.create_single_increasing_dataset(10, shape=[4, 4]) infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id()) infeed_queue.initializer with self.assertRaisesRegexp( ValueError, 'The IPUInfeedQueue `initializer` function can only be accessed once.' ): infeed_queue.initializer
def testCreateSimpleReplicatedInfeedOutfeed(self): shape = [2] dataset = tu.create_single_increasing_dataset(3, shape) infeed_queue = ipu_infeed_queue.IPUInfeedQueue( dataset, feed_name=next_feed_id(), replication_factor=2) outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue( feed_name=next_feed_id(), replication_factor=2) def body(v, x): v = popops_cross_replica_sum.cross_replica_sum(v + x) outfeed = outfeed_queue.enqueue(v) return (v, outfeed) def my_net(): v = constant_op.constant(0.0, shape=shape, dtype=np.float32) r = loops.repeat(5, body, [v], infeed_queue) return r with ipu.ops.ipu_scope("/device:IPU:0"): res = ipu_compiler.compile(my_net, inputs=[]) outfed = outfeed_queue.dequeue() cfg = ipu.utils.create_ipu_config( profiling=False, max_cross_replica_sum_buffer_size=10000) cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) cfg = ipu.utils.auto_select_ipus(cfg, 2) ipu.utils.configure_ipu_system(cfg) with sl.Session() as sess: sess.run(infeed_queue.initializer) result = sess.run(res) self.assertAllClose(result[0], np.broadcast_to(48, shape)) outfed_result = sess.run(outfed) self.assertTrue(outfed_result.shape[0], 2) self.assertAllClose(outfed_result[0][0], outfed_result[0][1]) self.assertAllClose(outfed_result[0][0], np.broadcast_to(1, shape)) self.assertAllClose(outfed_result[1][0], outfed_result[1][1]) self.assertAllClose(outfed_result[1][0], np.broadcast_to(4, shape)) self.assertAllClose(outfed_result[2][0], outfed_result[2][1]) self.assertAllClose(outfed_result[2][0], np.broadcast_to(11, shape)) self.assertAllClose(outfed_result[3][0], outfed_result[3][1]) self.assertAllClose(outfed_result[3][0], np.broadcast_to(23, shape)) self.assertAllClose(outfed_result[4][0], outfed_result[4][1]) self.assertAllClose(outfed_result[4][0], np.broadcast_to(48, shape))
def testTwoInfeedsDifferentPrograms(self): dataset1 = tu.create_single_increasing_dataset(20, shape=[4, 4]) dataset2 = tu.create_single_increasing_dataset(3, shape=[4, 4]) infeed_queue1 = ipu_infeed_queue.IPUInfeedQueue( dataset1, feed_name=next_feed_id()) infeed_queue2 = ipu_infeed_queue.IPUInfeedQueue( dataset2, feed_name=next_feed_id()) def program(iters, infeed_queue): def body(v, x): v = v + x return (v) def my_net(): v = constant_op.constant(0.0, shape=[4, 4], dtype=np.float32) r = loops.repeat(iters, body, (v), infeed_queue) return r with ipu.ops.ipu_scope("/device:IPU:0"): return ipu_compiler.compile(my_net) cfg = ipu.utils.create_ipu_config() cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) ipu.utils.configure_ipu_system(cfg) with session_lib.Session() as sess: sess.run(infeed_queue1.initializer) sess.run(infeed_queue2.initializer) result = sess.run(program(5, infeed_queue1)) self.assertAllClose(result[0], np.broadcast_to(10, [4, 4])) result = sess.run(program(5, infeed_queue2)) self.assertAllClose(result[0], np.broadcast_to(4, [4, 4])) result = sess.run(program(5, infeed_queue1)) self.assertAllClose(result[0], np.broadcast_to(35, [4, 4])) result = sess.run(program(5, infeed_queue2)) self.assertAllClose(result[0], np.broadcast_to(5, [4, 4]))
def testSingleInfeedOutfeedRepeatNamedLast(self): dataset = tu.create_single_increasing_dataset(3, shape=[4, 4]) shape = [4, 4] def dataset_parser(value): image_1 = value image_2 = (value + 10.) / 2.0 return (image_1, image_2) dataset = dataset.map(dataset_parser) infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id()) outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue( next_feed_id(), outfeed_mode=ipu_outfeed_queue.IPUOutfeedMode.LAST) def body(v, im1, im2): v = v + im1 + im2 outfeed = outfeed_queue.enqueue({ "v": v, "image1": im1, "image2": im2 }) return (v, outfeed) def my_net(): v = constant_op.constant(0.0, shape=shape, dtype=np.float32) r = loops.repeat(5, body, [v], infeed_queue) return r with ipu.ops.ipu_scope("/device:IPU:0"): res = ipu_compiler.compile(my_net, inputs=[]) outfed = outfeed_queue.dequeue() cfg = ipu.utils.create_ipu_config() cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) ipu.utils.configure_ipu_system(cfg) with session_lib.Session() as sess: sess.run(infeed_queue.initializer) result = sess.run(res) self.assertAllClose(result[0], np.broadcast_to(31, shape)) outfed_result = sess.run(outfed) self.assertTrue(len(outfed_result) == 3) self.assertAllClose(outfed_result["v"], np.broadcast_to(31, shape)) self.assertAllClose(outfed_result["image1"], np.broadcast_to(1, shape)) self.assertAllClose(outfed_result["image2"], np.broadcast_to(5.5, shape))
def testTrainingLoopWithInfeedAndOutfeedGetLast(self): dataset = tu.create_single_increasing_dataset(10, shape=[4, 4, 2]) dataset = dataset.batch(batch_size=2, drop_remainder=True) infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id()) outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue( next_feed_id(), outfeed_mode=ipu_outfeed_queue.IPUOutfeedMode.LAST) def my_net(iters): def body(loss, x): with variable_scope.variable_scope("vs", use_resource=True): y = layers.Conv2D( 2, 1, use_bias=True, kernel_initializer=init_ops.ones_initializer(), name='conv1')(x) loss = math_ops.reduce_sum(y) optimizer = gradient_descent.GradientDescentOptimizer(0.1) train = optimizer.minimize(loss) outfeed = outfeed_queue.enqueue(loss) with ops.control_dependencies([train]): return (array_ops.identity(loss), outfeed) loss = 0.0 return loops.repeat(iters, body, (loss), infeed_queue) with ops.device('cpu'): iters = array_ops.placeholder(np.int32, shape=[]) with ipu.ops.ipu_scope("/device:IPU:0"): r = ipu_compiler.compile(my_net, inputs=[iters]) outfeeds = outfeed_queue.dequeue() with session_lib.Session() as sess: sess.run(infeed_queue.initializer) sess.run(variables.global_variables_initializer()) initial_loss = sess.run(r, {iters: 1}) final_loss = sess.run(r, {iters: 1000}) outfed = sess.run(outfeeds) self.assertTrue(initial_loss > final_loss) self.assertTrue(outfed == final_loss) # Check that a scalar is returned instead of a numpy array self.assertTrue(type(outfed) == np.float32)
def testSingleInfeedWhileLoopTuple(self): dataset = tu.create_single_increasing_dataset(3, shape=[4, 4]) def dataset_parser(value): image_1 = value image_2 = (value + 10.) / 2.0 return (image_1, image_2) dataset = dataset.map(dataset_parser) infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id()) def cond(i, v): return i < 20 def body(i, v, im1, im2): v = v + im1 + im2 return (i + 1, v) def my_net(v): i = 0 r = loops.while_loop(cond, body, (i, v), infeed_queue) return r[1] with ops.device('cpu'): v = array_ops.placeholder(np.float32, [4, 4]) with ipu.ops.ipu_scope("/device:IPU:0"): res = ipu_compiler.compile(my_net, inputs=[v]) cfg = ipu.utils.create_ipu_config() cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) ipu.utils.configure_ipu_system(cfg) with session_lib.Session() as sess: sess.run(infeed_queue.initializer) result = sess.run(res, {v: np.ones([4, 4], np.float32)}) self.assertAllClose(result[0], np.broadcast_to(129.5, [4, 4]))
def testSingleInfeedRepeatNamed(self): dataset = tu.create_single_increasing_dataset(3, shape=[4, 4]) def dataset_parser(value): image_1 = value image_2 = (value + 10.) / 2.0 return {"a": image_1, "b": image_2} dataset = dataset.map(dataset_parser) infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id()) # Note how the parameters are swapped around. def body(v1, v2, b, a): v1 = v1 + a v2 = v2 + b return (v1, v2) def my_net(): v1 = constant_op.constant(0.0, shape=[4, 4], dtype=np.float32) v2 = constant_op.constant(0.0, shape=[4, 4], dtype=np.float32) r = loops.repeat(5, body, [v1, v2], infeed_queue) return r with ipu.ops.ipu_scope("/device:IPU:0"): res = ipu_compiler.compile(my_net, inputs=[]) cfg = ipu.utils.create_ipu_config() cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False) ipu.utils.configure_ipu_system(cfg) with session_lib.Session() as sess: sess.run(infeed_queue.initializer) result = sess.run(res) self.assertAllClose(result[0], np.broadcast_to(4, [4, 4])) self.assertAllClose(result[1], np.broadcast_to(27, [4, 4]))
def testUndefinedShape(self): dataset = tu.create_single_increasing_dataset(10, shape=[4, 4]) dataset = dataset.batch(10, drop_remainder=False) with self.assertRaisesRegexp(ValueError, 'Output shape \(\?,'): infeed_queue = ipu_infeed_queue.IPUInfeedQueue( dataset, next_feed_id())