def testSingleInfeedMultipleRuns(self):
dataset = tu.create_single_increasing_dataset(10, shape=[4, 4])
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id())
def program(iters):
def body(v, x):
v = v + x
return (v)
def my_net():
v = constant_op.constant(0.0, shape=[4, 4], dtype=np.float32)
r = loops.repeat(iters, body, (v), infeed_queue)
return r
with ipu.ops.ipu_scope("/device:IPU:0"):
return ipu_compiler.compile(my_net)
cfg = ipu.utils.create_ipu_config()
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
ipu.utils.configure_ipu_system(cfg)
with session_lib.Session() as sess:
sess.run(infeed_queue.initializer)
result = sess.run(program(0))
self.assertAllClose(result[0], np.broadcast_to(0, [4, 4]))
# The iterator has not moved - next element should be all 1s.
result = sess.run(program(2))
self.assertAllClose(result[0], np.broadcast_to(1, [4, 4]))
# The iterator has moved - in the next two iterations it should pull 2 and 3.
result = sess.run(program(2))
self.assertAllClose(result[0], np.broadcast_to(5, [4, 4]))
# The iterator has moved - in the next two iterations it should pull 4 and 5.
result = sess.run(program(2))
self.assertAllClose(result[0], np.broadcast_to(9, [4, 4]))
def testSingleInfeedRepeatNonTupleFiniteDataset(self):
dataset = tu.create_single_increasing_dataset(10,
shape=[4, 4],
repeat=False)
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id())
def body(v, x):
v = v + x
return (v)
def my_net(v):
r = loops.repeat(10, body, (v), infeed_queue)
return r
with ops.device('cpu'):
v = array_ops.placeholder(np.float32, [4, 4])
with ipu.ops.ipu_scope("/device:IPU:0"):
res = ipu_compiler.compile(my_net, inputs=[v])
cfg = ipu.utils.create_ipu_config()
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
ipu.utils.configure_ipu_system(cfg)
with session_lib.Session() as sess:
sess.run(infeed_queue.initializer)
result = sess.run(res, {v: np.ones([4, 4], np.float32)})
self.assertAllClose(result[0], np.broadcast_to(46, [4, 4]))
def testSingleInfeedMultipleRepeats(self):
dataset = tu.create_single_increasing_dataset(2, shape=[4, 4])
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id())
def body(v, x):
v = v + x
return (v)
def my_net():
v = constant_op.constant(0.0, shape=[4, 4], dtype=np.float32)
r = loops.repeat(5, body, [v], infeed_queue)
r = loops.repeat(5, body, [r], infeed_queue)
return r
with ipu.ops.ipu_scope("/device:IPU:0"):
res = ipu_compiler.compile(my_net, inputs=[])
cfg = ipu.utils.create_ipu_config()
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
ipu.utils.configure_ipu_system(cfg)
with session_lib.Session() as sess:
sess.run(infeed_queue.initializer)
result = sess.run(res)
self.assertAllClose(result[0], np.broadcast_to(5, [4, 4]))
def testSingleInfeedRepeatTupleMerge(self):
dataset = tu.create_single_increasing_dataset(3, shape=[4, 4])
def dataset_parser(value):
image_1 = value
image_2 = (value + 10.) / 2.0
return (image_1, image_2)
dataset = dataset.map(dataset_parser)
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id())
def body(v, im1, im2):
v = v + im1 + im2
return (v)
def my_net():
v = constant_op.constant(0.0, shape=[4, 4], dtype=np.float32)
r = loops.repeat(5, body, [v], infeed_queue)
return r
with ipu.ops.ipu_scope("/device:IPU:0"):
res = ipu_compiler.compile(my_net, inputs=[])
cfg = ipu.utils.create_ipu_config(merge_infeed_io_copies=True)
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
ipu.utils.configure_ipu_system(cfg)
with session_lib.Session() as sess:
sess.run(infeed_queue.initializer)
result = sess.run(res)
self.assertAllClose(result[0], np.broadcast_to(31, [4, 4]))
def testCreateSimpleReplicatedOutfeedWrongReplicationFactor(self):
shape = [2]
dataset = tu.create_single_increasing_dataset(3, shape)
outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue(
feed_name=next_feed_id(), replication_factor=4)
def body(v):
v = popops_cross_replica_sum.cross_replica_sum(v)
outfeed = outfeed_queue.enqueue(v)
return (v, outfeed)
def my_net():
v = constant_op.constant(0.0, shape=shape, dtype=np.float32)
r = loops.repeat(5, body, [v])
return r
with ipu.ops.ipu_scope("/device:IPU:0"):
res = ipu_compiler.compile(my_net, inputs=[])
cfg = ipu.utils.create_ipu_config(
profiling=False, max_cross_replica_sum_buffer_size=10000)
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
cfg = ipu.utils.auto_select_ipus(cfg, 2)
ipu.utils.configure_ipu_system(cfg)
with sl.Session() as sess:
with self.assertRaisesRegexp(
errors.FailedPreconditionError,
'Current program has been created with replication_factor 2'
):
result = sess.run(res)
def testErrorWhenNoAllReduce(self):
shape = [2]
dataset = tu.create_single_increasing_dataset(3, shape)
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(
dataset, feed_name=next_feed_id(), replication_factor=2)
outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue(
feed_name=next_feed_id(), replication_factor=2)
def body(v, x):
outfeed = outfeed_queue.enqueue(v)
return (v + x, outfeed)
def my_net():
v = constant_op.constant(0.0, shape=shape, dtype=np.float32)
r = loops.repeat(5, body, [v], infeed_queue)
return r
with ipu.ops.ipu_scope("/device:IPU:0"):
res = ipu_compiler.compile(my_net, inputs=[])
outfed = outfeed_queue.dequeue()
cfg = ipu.utils.create_ipu_config(
profiling=False, max_cross_replica_sum_buffer_size=10000)
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
cfg = ipu.utils.auto_select_ipus(cfg, 2)
ipu.utils.configure_ipu_system(cfg)
with sl.Session() as sess:
sess.run(infeed_queue.initializer)
with self.assertRaisesRegexp(
errors.FailedPreconditionError,
'This is not a valid replicated graph because'):
result = sess.run(res)
def testMultipleInitializations(self):
dataset = tu.create_single_increasing_dataset(10, shape=[4, 4])
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id())
infeed_queue.initializer
with self.assertRaisesRegexp(
ValueError,
'The IPUInfeedQueue `initializer` function can only be accessed once.'
):
infeed_queue.initializer
def testCreateSimpleReplicatedInfeedOutfeed(self):
shape = [2]
dataset = tu.create_single_increasing_dataset(3, shape)
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(
dataset, feed_name=next_feed_id(), replication_factor=2)
outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue(
feed_name=next_feed_id(), replication_factor=2)
def body(v, x):
v = popops_cross_replica_sum.cross_replica_sum(v + x)
outfeed = outfeed_queue.enqueue(v)
return (v, outfeed)
def my_net():
v = constant_op.constant(0.0, shape=shape, dtype=np.float32)
r = loops.repeat(5, body, [v], infeed_queue)
return r
with ipu.ops.ipu_scope("/device:IPU:0"):
res = ipu_compiler.compile(my_net, inputs=[])
outfed = outfeed_queue.dequeue()
cfg = ipu.utils.create_ipu_config(
profiling=False, max_cross_replica_sum_buffer_size=10000)
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
cfg = ipu.utils.auto_select_ipus(cfg, 2)
ipu.utils.configure_ipu_system(cfg)
with sl.Session() as sess:
sess.run(infeed_queue.initializer)
result = sess.run(res)
self.assertAllClose(result[0], np.broadcast_to(48, shape))
outfed_result = sess.run(outfed)
self.assertTrue(outfed_result.shape[0], 2)
self.assertAllClose(outfed_result[0][0], outfed_result[0][1])
self.assertAllClose(outfed_result[0][0], np.broadcast_to(1, shape))
self.assertAllClose(outfed_result[1][0], outfed_result[1][1])
self.assertAllClose(outfed_result[1][0], np.broadcast_to(4, shape))
self.assertAllClose(outfed_result[2][0], outfed_result[2][1])
self.assertAllClose(outfed_result[2][0],
np.broadcast_to(11, shape))
self.assertAllClose(outfed_result[3][0], outfed_result[3][1])
self.assertAllClose(outfed_result[3][0],
np.broadcast_to(23, shape))
self.assertAllClose(outfed_result[4][0], outfed_result[4][1])
self.assertAllClose(outfed_result[4][0],
np.broadcast_to(48, shape))
def testTwoInfeedsDifferentPrograms(self):
dataset1 = tu.create_single_increasing_dataset(20, shape=[4, 4])
dataset2 = tu.create_single_increasing_dataset(3, shape=[4, 4])
infeed_queue1 = ipu_infeed_queue.IPUInfeedQueue(
dataset1, feed_name=next_feed_id())
infeed_queue2 = ipu_infeed_queue.IPUInfeedQueue(
dataset2, feed_name=next_feed_id())
def program(iters, infeed_queue):
def body(v, x):
v = v + x
return (v)
def my_net():
v = constant_op.constant(0.0, shape=[4, 4], dtype=np.float32)
r = loops.repeat(iters, body, (v), infeed_queue)
return r
with ipu.ops.ipu_scope("/device:IPU:0"):
return ipu_compiler.compile(my_net)
cfg = ipu.utils.create_ipu_config()
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
ipu.utils.configure_ipu_system(cfg)
with session_lib.Session() as sess:
sess.run(infeed_queue1.initializer)
sess.run(infeed_queue2.initializer)
result = sess.run(program(5, infeed_queue1))
self.assertAllClose(result[0], np.broadcast_to(10, [4, 4]))
result = sess.run(program(5, infeed_queue2))
self.assertAllClose(result[0], np.broadcast_to(4, [4, 4]))
result = sess.run(program(5, infeed_queue1))
self.assertAllClose(result[0], np.broadcast_to(35, [4, 4]))
result = sess.run(program(5, infeed_queue2))
self.assertAllClose(result[0], np.broadcast_to(5, [4, 4]))
def testSingleInfeedOutfeedRepeatNamedLast(self):
dataset = tu.create_single_increasing_dataset(3, shape=[4, 4])
shape = [4, 4]
def dataset_parser(value):
image_1 = value
image_2 = (value + 10.) / 2.0
return (image_1, image_2)
dataset = dataset.map(dataset_parser)
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id())
outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue(
next_feed_id(), outfeed_mode=ipu_outfeed_queue.IPUOutfeedMode.LAST)
def body(v, im1, im2):
v = v + im1 + im2
outfeed = outfeed_queue.enqueue({
"v": v,
"image1": im1,
"image2": im2
})
return (v, outfeed)
def my_net():
v = constant_op.constant(0.0, shape=shape, dtype=np.float32)
r = loops.repeat(5, body, [v], infeed_queue)
return r
with ipu.ops.ipu_scope("/device:IPU:0"):
res = ipu_compiler.compile(my_net, inputs=[])
outfed = outfeed_queue.dequeue()
cfg = ipu.utils.create_ipu_config()
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
ipu.utils.configure_ipu_system(cfg)
with session_lib.Session() as sess:
sess.run(infeed_queue.initializer)
result = sess.run(res)
self.assertAllClose(result[0], np.broadcast_to(31, shape))
outfed_result = sess.run(outfed)
self.assertTrue(len(outfed_result) == 3)
self.assertAllClose(outfed_result["v"], np.broadcast_to(31, shape))
self.assertAllClose(outfed_result["image1"],
np.broadcast_to(1, shape))
self.assertAllClose(outfed_result["image2"],
np.broadcast_to(5.5, shape))
def testTrainingLoopWithInfeedAndOutfeedGetLast(self):
dataset = tu.create_single_increasing_dataset(10, shape=[4, 4, 2])
dataset = dataset.batch(batch_size=2, drop_remainder=True)
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id())
outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue(
next_feed_id(), outfeed_mode=ipu_outfeed_queue.IPUOutfeedMode.LAST)
def my_net(iters):
def body(loss, x):
with variable_scope.variable_scope("vs", use_resource=True):
y = layers.Conv2D(
2,
1,
use_bias=True,
kernel_initializer=init_ops.ones_initializer(),
name='conv1')(x)
loss = math_ops.reduce_sum(y)
optimizer = gradient_descent.GradientDescentOptimizer(0.1)
train = optimizer.minimize(loss)
outfeed = outfeed_queue.enqueue(loss)
with ops.control_dependencies([train]):
return (array_ops.identity(loss), outfeed)
loss = 0.0
return loops.repeat(iters, body, (loss), infeed_queue)
with ops.device('cpu'):
iters = array_ops.placeholder(np.int32, shape=[])
with ipu.ops.ipu_scope("/device:IPU:0"):
r = ipu_compiler.compile(my_net, inputs=[iters])
outfeeds = outfeed_queue.dequeue()
with session_lib.Session() as sess:
sess.run(infeed_queue.initializer)
sess.run(variables.global_variables_initializer())
initial_loss = sess.run(r, {iters: 1})
final_loss = sess.run(r, {iters: 1000})
outfed = sess.run(outfeeds)
self.assertTrue(initial_loss > final_loss)
self.assertTrue(outfed == final_loss)
# Check that a scalar is returned instead of a numpy array
self.assertTrue(type(outfed) == np.float32)
def testSingleInfeedWhileLoopTuple(self):
dataset = tu.create_single_increasing_dataset(3, shape=[4, 4])
def dataset_parser(value):
image_1 = value
image_2 = (value + 10.) / 2.0
return (image_1, image_2)
dataset = dataset.map(dataset_parser)
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id())
def cond(i, v):
return i < 20
def body(i, v, im1, im2):
v = v + im1 + im2
return (i + 1, v)
def my_net(v):
i = 0
r = loops.while_loop(cond, body, (i, v), infeed_queue)
return r[1]
with ops.device('cpu'):
v = array_ops.placeholder(np.float32, [4, 4])
with ipu.ops.ipu_scope("/device:IPU:0"):
res = ipu_compiler.compile(my_net, inputs=[v])
cfg = ipu.utils.create_ipu_config()
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
ipu.utils.configure_ipu_system(cfg)
with session_lib.Session() as sess:
sess.run(infeed_queue.initializer)
result = sess.run(res, {v: np.ones([4, 4], np.float32)})
self.assertAllClose(result[0], np.broadcast_to(129.5, [4, 4]))
def testSingleInfeedRepeatNamed(self):
dataset = tu.create_single_increasing_dataset(3, shape=[4, 4])
def dataset_parser(value):
image_1 = value
image_2 = (value + 10.) / 2.0
return {"a": image_1, "b": image_2}
dataset = dataset.map(dataset_parser)
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id())
# Note how the parameters are swapped around.
def body(v1, v2, b, a):
v1 = v1 + a
v2 = v2 + b
return (v1, v2)
def my_net():
v1 = constant_op.constant(0.0, shape=[4, 4], dtype=np.float32)
v2 = constant_op.constant(0.0, shape=[4, 4], dtype=np.float32)
r = loops.repeat(5, body, [v1, v2], infeed_queue)
return r
with ipu.ops.ipu_scope("/device:IPU:0"):
res = ipu_compiler.compile(my_net, inputs=[])
cfg = ipu.utils.create_ipu_config()
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
ipu.utils.configure_ipu_system(cfg)
with session_lib.Session() as sess:
sess.run(infeed_queue.initializer)
result = sess.run(res)
self.assertAllClose(result[0], np.broadcast_to(4, [4, 4]))
self.assertAllClose(result[1], np.broadcast_to(27, [4, 4]))
def testUndefinedShape(self):
dataset = tu.create_single_increasing_dataset(10, shape=[4, 4])
dataset = dataset.batch(10, drop_remainder=False)
with self.assertRaisesRegexp(ValueError, 'Output shape \(\?,'):
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(
dataset, next_feed_id())
