def testMultipleOutfeedsRepeatNonTuple(self):
outfeed_queue1 = ipu_outfeed_queue.IPUOutfeedQueue(next_feed_id())
outfeed_queue2 = ipu_outfeed_queue.IPUOutfeedQueue(next_feed_id())
def body(v):
outfeed1 = outfeed_queue1.enqueue(v)
outfeed2 = outfeed_queue2.enqueue(v * 2)
v = v + 1
return (v, outfeed1, outfeed2)
def my_net(v):
r = loops.repeat(20, body, (v))
return r
with ops.device('cpu'):
v = array_ops.placeholder(np.float32, [4, 4])
with ipu.ops.ipu_scope("/device:IPU:0"):
res = ipu_compiler.compile(my_net, inputs=[v])
cfg = ipu.utils.create_ipu_config()
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
ipu.utils.configure_ipu_system(cfg)
outfeed1 = outfeed_queue1.dequeue()
outfeed2 = outfeed_queue2.dequeue()
with session_lib.Session() as sess:
with self.assertRaisesRegexp(
errors.InvalidArgumentError,
'Only one IPUOutfeedQueue supported per graph'):
result = sess.run(res, {v: np.ones([4, 4], np.float32)})
def testTwoOutfeedsDifferentPrograms(self):
outfeed_queue1 = ipu_outfeed_queue.IPUOutfeedQueue(
feed_name=next_feed_id())
outfeed_queue2 = ipu_outfeed_queue.IPUOutfeedQueue(
feed_name=next_feed_id())
def body1(v):
outfeed = outfeed_queue1.enqueue(v)
v = v + 1
return (v, outfeed)
def my_net1(v):
r = loops.repeat(5, body1, (v))
return r
def body2(v):
outfeed = outfeed_queue2.enqueue(v)
v = v + 1
return (v, outfeed)
def my_net2(v):
r = loops.repeat(7, body2, (v))
return r
with ops.device('cpu'):
v1 = array_ops.placeholder(np.float32, [4, 4])
v2 = array_ops.placeholder(np.float32, [5, 5])
with ipu.ops.ipu_scope("/device:IPU:0"):
res1 = ipu_compiler.compile(my_net1, inputs=[v1])
res2 = ipu_compiler.compile(my_net2, inputs=[v2])
cfg = ipu.utils.create_ipu_config()
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
ipu.utils.configure_ipu_system(cfg)
outfeed1 = outfeed_queue1.dequeue()
outfeed2 = outfeed_queue2.dequeue()
with session_lib.Session() as sess:
result1 = sess.run(res1, {v1: np.ones([4, 4], np.float32)})
self.assertAllClose(result1[0], np.broadcast_to(6, [4, 4]))
outfed1 = sess.run(outfeed1)
for i in range(5):
self.assertAllClose(outfed1[i], np.broadcast_to(i + 1, [4, 4]))
result2 = sess.run(res2, {v2: np.full([5, 5], 4, np.float32)})
self.assertAllClose(result2[0], np.broadcast_to(11, [5, 5]))
outfed2 = sess.run(outfeed2)
for i in range(7):
self.assertAllClose(outfed2[i], np.broadcast_to(i + 4, [5, 5]))
def testSingleInfeedOutfeedRepeatNonTuple(self):
dataset = tu.create_single_increasing_dataset(10, shape=[4, 4])
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id())
outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue(next_feed_id())
def body(v, x):
v = v + x
outfeed = outfeed_queue.enqueue(v)
return (v, outfeed)
def my_net(v):
r = loops.repeat(20, body, (v), infeed_queue)
return r
with ops.device('cpu'):
v = array_ops.placeholder(np.float32, [4, 4])
with ipu.ops.ipu_scope("/device:IPU:0"):
res = ipu_compiler.compile(my_net, inputs=[v])
cfg = ipu.utils.create_ipu_config()
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
ipu.utils.configure_ipu_system(cfg)
with session_lib.Session() as sess:
sess.run(infeed_queue.initializer)
result = sess.run(res, {v: np.ones([4, 4], np.float32)})
self.assertAllClose(result[0], np.broadcast_to(91, [4, 4]))
outfed = sess.run(outfeed_queue.dequeue())
self.assertEqual(outfed.shape, (20, 4, 4))
self.assertAllClose(outfed[-1], result[0])
self.assertAllClose(outfed[5], np.broadcast_to(16, [4, 4]))
def testSingleOutfeedRepeatNonTuple(self):
outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue(next_feed_id())
def body(v):
outfeed = outfeed_queue.enqueue(v)
v = v + 1
return (v, outfeed)
def my_net(v):
r = loops.repeat(20, body, (v))
return r
with ops.device('cpu'):
v = array_ops.placeholder(np.float32, [4, 4])
with ipu.ops.ipu_scope("/device:IPU:0"):
res = ipu_compiler.compile(my_net, inputs=[v])
cfg = ipu.utils.create_ipu_config()
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
ipu.utils.configure_ipu_system(cfg)
outfeed = outfeed_queue.dequeue()
with session_lib.Session() as sess:
result = sess.run(res, {v: np.ones([4, 4], np.float32)})
self.assertAllClose(result[0], np.broadcast_to(21, [4, 4]))
outfed = sess.run(outfeed)
for i in range(20):
self.assertAllClose(outfed[i], np.broadcast_to(i + 1, [4, 4]))
def testCreateSimpleReplicatedOutfeedWrongReplicationFactor(self):
shape = [2]
dataset = tu.create_single_increasing_dataset(3, shape)
outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue(
feed_name=next_feed_id(), replication_factor=4)
def body(v):
v = popops_cross_replica_sum.cross_replica_sum(v)
outfeed = outfeed_queue.enqueue(v)
return (v, outfeed)
def my_net():
v = constant_op.constant(0.0, shape=shape, dtype=np.float32)
r = loops.repeat(5, body, [v])
return r
with ipu.ops.ipu_scope("/device:IPU:0"):
res = ipu_compiler.compile(my_net, inputs=[])
cfg = ipu.utils.create_ipu_config(
profiling=False, max_cross_replica_sum_buffer_size=10000)
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
cfg = ipu.utils.auto_select_ipus(cfg, 2)
ipu.utils.configure_ipu_system(cfg)
with sl.Session() as sess:
with self.assertRaisesRegexp(
errors.FailedPreconditionError,
'Current program has been created with replication_factor 2'
):
result = sess.run(res)
def testErrorWhenNoAllReduce(self):
shape = [2]
dataset = tu.create_single_increasing_dataset(3, shape)
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(
dataset, feed_name=next_feed_id(), replication_factor=2)
outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue(
feed_name=next_feed_id(), replication_factor=2)
def body(v, x):
outfeed = outfeed_queue.enqueue(v)
return (v + x, outfeed)
def my_net():
v = constant_op.constant(0.0, shape=shape, dtype=np.float32)
r = loops.repeat(5, body, [v], infeed_queue)
return r
with ipu.ops.ipu_scope("/device:IPU:0"):
res = ipu_compiler.compile(my_net, inputs=[])
outfed = outfeed_queue.dequeue()
cfg = ipu.utils.create_ipu_config(
profiling=False, max_cross_replica_sum_buffer_size=10000)
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
cfg = ipu.utils.auto_select_ipus(cfg, 2)
ipu.utils.configure_ipu_system(cfg)
with sl.Session() as sess:
sess.run(infeed_queue.initializer)
with self.assertRaisesRegexp(
errors.FailedPreconditionError,
'This is not a valid replicated graph because'):
result = sess.run(res)
def testTwoOutfeedsDifferentProgramsSameFeedName(self):
outfeed_queue1 = ipu_outfeed_queue.IPUOutfeedQueue(feed_name="a")
outfeed_queue2 = ipu_outfeed_queue.IPUOutfeedQueue(feed_name="a")
def body1(v):
outfeed = outfeed_queue1.enqueue(v)
v = v + 1
return (v, outfeed)
def my_net1(v):
r = loops.repeat(5, body1, (v))
return r
def body2(v):
outfeed = outfeed_queue2.enqueue(v)
v = v + 1
return (v, outfeed)
def my_net2(v):
r = loops.repeat(7, body2, (v))
return r
with ops.device('cpu'):
v1 = array_ops.placeholder(np.float32, [4, 4])
v2 = array_ops.placeholder(np.float32, [5, 5])
with ipu.ops.ipu_scope("/device:IPU:0"):
res1 = ipu_compiler.compile(my_net1, inputs=[v1])
res2 = ipu_compiler.compile(my_net2, inputs=[v2])
cfg = ipu.utils.create_ipu_config()
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
ipu.utils.configure_ipu_system(cfg)
outfeed1 = outfeed_queue1.dequeue()
outfeed2 = outfeed_queue2.dequeue()
with session_lib.Session() as sess:
result1 = sess.run(res1, {v1: np.ones([4, 4], np.float32)})
with self.assertRaisesRegexp(
errors.FailedPreconditionError,
'Outfeed with id=\'a\' already exists'):
result2 = sess.run(res2, {v2: np.full([5, 5], 4, np.float32)})
def testCreateSimpleReplicatedInfeedOutfeed(self):
shape = [2]
dataset = tu.create_single_increasing_dataset(3, shape)
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(
dataset, feed_name=next_feed_id(), replication_factor=2)
outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue(
feed_name=next_feed_id(), replication_factor=2)
def body(v, x):
v = popops_cross_replica_sum.cross_replica_sum(v + x)
outfeed = outfeed_queue.enqueue(v)
return (v, outfeed)
def my_net():
v = constant_op.constant(0.0, shape=shape, dtype=np.float32)
r = loops.repeat(5, body, [v], infeed_queue)
return r
with ipu.ops.ipu_scope("/device:IPU:0"):
res = ipu_compiler.compile(my_net, inputs=[])
outfed = outfeed_queue.dequeue()
cfg = ipu.utils.create_ipu_config(
profiling=False, max_cross_replica_sum_buffer_size=10000)
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
cfg = ipu.utils.auto_select_ipus(cfg, 2)
ipu.utils.configure_ipu_system(cfg)
with sl.Session() as sess:
sess.run(infeed_queue.initializer)
result = sess.run(res)
self.assertAllClose(result[0], np.broadcast_to(48, shape))
outfed_result = sess.run(outfed)
self.assertTrue(outfed_result.shape[0], 2)
self.assertAllClose(outfed_result[0][0], outfed_result[0][1])
self.assertAllClose(outfed_result[0][0], np.broadcast_to(1, shape))
self.assertAllClose(outfed_result[1][0], outfed_result[1][1])
self.assertAllClose(outfed_result[1][0], np.broadcast_to(4, shape))
self.assertAllClose(outfed_result[2][0], outfed_result[2][1])
self.assertAllClose(outfed_result[2][0],
np.broadcast_to(11, shape))
self.assertAllClose(outfed_result[3][0], outfed_result[3][1])
self.assertAllClose(outfed_result[3][0],
np.broadcast_to(23, shape))
self.assertAllClose(outfed_result[4][0], outfed_result[4][1])
self.assertAllClose(outfed_result[4][0],
np.broadcast_to(48, shape))
def testSingleInfeedOutfeedRepeatNamedLast(self):
dataset = tu.create_single_increasing_dataset(3, shape=[4, 4])
shape = [4, 4]
def dataset_parser(value):
image_1 = value
image_2 = (value + 10.) / 2.0
return (image_1, image_2)
dataset = dataset.map(dataset_parser)
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id())
outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue(
next_feed_id(), outfeed_mode=ipu_outfeed_queue.IPUOutfeedMode.LAST)
def body(v, im1, im2):
v = v + im1 + im2
outfeed = outfeed_queue.enqueue({
"v": v,
"image1": im1,
"image2": im2
})
return (v, outfeed)
def my_net():
v = constant_op.constant(0.0, shape=shape, dtype=np.float32)
r = loops.repeat(5, body, [v], infeed_queue)
return r
with ipu.ops.ipu_scope("/device:IPU:0"):
res = ipu_compiler.compile(my_net, inputs=[])
outfed = outfeed_queue.dequeue()
cfg = ipu.utils.create_ipu_config()
cfg = ipu.utils.set_ipu_model_options(cfg, compile_ipu_code=False)
ipu.utils.configure_ipu_system(cfg)
with session_lib.Session() as sess:
sess.run(infeed_queue.initializer)
result = sess.run(res)
self.assertAllClose(result[0], np.broadcast_to(31, shape))
outfed_result = sess.run(outfed)
self.assertTrue(len(outfed_result) == 3)
self.assertAllClose(outfed_result["v"], np.broadcast_to(31, shape))
self.assertAllClose(outfed_result["image1"],
np.broadcast_to(1, shape))
self.assertAllClose(outfed_result["image2"],
np.broadcast_to(5.5, shape))
def testTrainingLoopWithInfeedAndOutfeedGetLast(self):
dataset = tu.create_single_increasing_dataset(10, shape=[4, 4, 2])
dataset = dataset.batch(batch_size=2, drop_remainder=True)
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, next_feed_id())
outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue(
next_feed_id(), outfeed_mode=ipu_outfeed_queue.IPUOutfeedMode.LAST)
def my_net(iters):
def body(loss, x):
with variable_scope.variable_scope("vs", use_resource=True):
y = layers.Conv2D(
2,
1,
use_bias=True,
kernel_initializer=init_ops.ones_initializer(),
name='conv1')(x)
loss = math_ops.reduce_sum(y)
optimizer = gradient_descent.GradientDescentOptimizer(0.1)
train = optimizer.minimize(loss)
outfeed = outfeed_queue.enqueue(loss)
with ops.control_dependencies([train]):
return (array_ops.identity(loss), outfeed)
loss = 0.0
return loops.repeat(iters, body, (loss), infeed_queue)
with ops.device('cpu'):
iters = array_ops.placeholder(np.int32, shape=[])
with ipu.ops.ipu_scope("/device:IPU:0"):
r = ipu_compiler.compile(my_net, inputs=[iters])
outfeeds = outfeed_queue.dequeue()
with session_lib.Session() as sess:
sess.run(infeed_queue.initializer)
sess.run(variables.global_variables_initializer())
initial_loss = sess.run(r, {iters: 1})
final_loss = sess.run(r, {iters: 1000})
outfed = sess.run(outfeeds)
self.assertTrue(initial_loss > final_loss)
self.assertTrue(outfed == final_loss)
# Check that a scalar is returned instead of a numpy array
self.assertTrue(type(outfed) == np.float32)
