def testPipelineIterationsNotMultiple(self):
dataset = tu.create_single_increasing_dataset(5, shape=[4, 4, 2])
dataset = dataset.batch(batch_size=2, drop_remainder=True)
def dataset_parser(value):
a = value
b = (value + 10.) / 2.0
return {"a": a, "b": b}
dataset = dataset.map(dataset_parser)
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, "__feed1")
outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue("__feed1")
def stage1(c, **kwargs):
with variable_scope.variable_scope("vs", use_resource=True):
y = layers.Conv2D(
2,
1,
use_bias=True,
kernel_initializer=init_ops.ones_initializer(),
name='conv1')(kwargs["a"])
return y + kwargs["b"], c
def stage2(x, c):
return math_ops.reduce_sum(x) + c
def stage3(x):
return x
def my_net(c):
return pipelining_ops.pipeline(
[stage1, stage2, stage3],
10,
inputs=[c],
infeed_queue=infeed_queue,
outfeed_queue=outfeed_queue,
pipeline_schedule=pipelining_ops.PipelineSchedule.Grouped)
with ops.device('cpu'):
c = array_ops.placeholder(np.float32, shape=[])
with tu.ipu_session() as sess:
with ops.device("/device:IPU:0"):
r = ipu_compiler.compile(my_net, inputs=[c])
cfg = utils.create_ipu_config(profiling=True,
profile_execution=True)
cfg = utils.auto_select_ipus(cfg, 4)
utils.configure_ipu_system(cfg)
utils.move_variable_initialization_to_cpu()
sess.run(variables.global_variables_initializer())
sess.run(infeed_queue.initializer)
with self.assertRaisesRegex(
errors.FailedPreconditionError,
'The pipeline depth of the pipeline must be a multiple of 3'
):
sess.run(r, {c: 10.01})
def dataset_fn():
dataset = tu.create_single_increasing_dataset(7, shape=[4, 4, 2])
dataset = dataset.batch(batch_size=2, drop_remainder=True)
def dataset_parser(value):
img = value / 7
label = value[0][0][0][0]
return img, label
return dataset.map(dataset_parser)
def dataset_fn():
dataset = tu.create_single_increasing_dataset(10, shape=[4])
dataset = dataset.batch(batch_size=2, drop_remainder=True)
def dataset_parser(value):
label = math_ops.reduce_mean(value, axis=[1])
return math_ops.cast(value, np.int32), math_ops.cast(
label / 10, np.int32)
return dataset.map(dataset_parser)
def dataset_fn():
dataset = tu.create_single_increasing_dataset(7, shape=[4, 4])
def dataset_parser(value):
img = value
label = value[0][0] % 4
return img, math_ops.cast(label, np.int32)
dataset = dataset.map(dataset_parser)
return dataset.batch(batch_size=2, drop_remainder=True)
def dataset_fn():
dataset = tu.create_single_increasing_dataset(100, shape=[4])
dataset = dataset.batch(batch_size=32, drop_remainder=True)
dataset = dataset.batch(batch_size=32, drop_remainder=True)
dataset = dataset.batch(batch_size=2, drop_remainder=True)
def dataset_parser(value):
img = value
label = math_ops.reduce_mean(img, axis=[1, 2, 3])
return img, math_ops.cast(label, np.int32)
return dataset.map(dataset_parser)
def testBufferDataset(self):
dataset = tu.create_single_increasing_dataset(10, shape=[4, 4])
dataset = dataset.take(3)
dataset = ipu.data.ops.dataset_ops.BufferDataset(dataset, 2)
itr = compat_v1_data.make_one_shot_iterator(dataset)
next_data = itr.get_next()
with self.session() as sess:
self.assertAllEqual(sess.run(next_data)[0], np.zeros([4, 4]))
self.assertAllEqual(sess.run(next_data)[0], np.ones([4, 4]))
with self.assertRaises(errors.OutOfRangeError):
self.evaluate(sess.run(next_data))
def testPipelineWithInfeedsKwargs(self):
with tu.ipu_session() as sess:
dataset = tu.create_single_increasing_dataset(5, shape=[4, 4, 2])
dataset = dataset.batch(batch_size=2, drop_remainder=True)
def dataset_parser(value):
a = value
b = (value + 10.) / 2.0
return {"a": a, "b": b}
dataset = dataset.map(dataset_parser)
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, "__feed6")
outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue("__feed6")
def stage1(c, **kwargs):
with variable_scope.variable_scope("vs", use_resource=True):
y = layers.Conv2D(2,
1,
use_bias=True,
kernel_initializer=init_ops.ones_initializer(),
name='conv1')(kwargs["a"])
return y + kwargs["b"], c
def stage2(x, c):
return math_ops.reduce_sum(x) + c
def stage3(x):
return x
def my_net(c):
return pipelining_ops.pipeline(
[stage1, stage2, stage3],
12,
inputs=[c],
infeed_queue=infeed_queue,
outfeed_queue=outfeed_queue,
pipeline_schedule=pipelining_ops.PipelineSchedule.Sequential)
with ops.device('cpu'):
c = array_ops.placeholder(np.float32, shape=[])
with ops.device("/device:IPU:0"):
r = ipu_compiler.compile(my_net, inputs=[c])
cfg = utils.create_ipu_config(profiling=True, profile_execution=True)
cfg = utils.auto_select_ipus(cfg, 4)
utils.configure_ipu_system(cfg)
utils.move_variable_initialization_to_cpu()
outfeed_op = outfeed_queue.dequeue()
report = tu.ReportJSON(self, sess, configure_device=False)
report.reset()
sess.run(variables.global_variables_initializer())
sess.run(infeed_queue.initializer)
sess.run(r, {c: 10.01})
losses_pipeline = sess.run(outfeed_op)
self.assertAllClose(losses_pipeline, [[
410.01, 730.01, 650.01, 570.01, 890.01, 410.01, 730.01, 650.01,
570.01, 890.01, 410.01, 730.01
]])
report.parse_log()
report.assert_pipeline_stages_on_expected_ipu((0, 1, 3))
def testPipelineInvalidDeviceMapping(self):
dataset = tu.create_single_increasing_dataset(5, shape=[4, 4, 2])
dataset = dataset.batch(batch_size=2, drop_remainder=True)
def dataset_parser(value):
a = value
b = (value + 10.) / 2.0
return {"a": a, "b": b}
dataset = dataset.map(dataset_parser)
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, "__feed3")
outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue("__feed3")
def stage1(c, **kwargs):
with variable_scope.variable_scope("vs", use_resource=True):
y = layers.Conv2D(2,
1,
use_bias=True,
kernel_initializer=init_ops.ones_initializer(),
name='conv1')(kwargs["a"])
return y + kwargs["b"], c
def stage2(x, c):
return math_ops.reduce_sum(x) + c
def stage3(x):
return x
with ops.device('cpu'):
c = array_ops.placeholder(np.float32, shape=[])
# Wrong type:
with self.assertRaisesRegex(
TypeError, 'device_mapping argument needs to be a list or a tuple'):
pipelining_ops.pipeline(
[stage1, stage2, stage3],
3,
inputs=[c],
infeed_queue=infeed_queue,
outfeed_queue=outfeed_queue,
device_mapping=1,
pipeline_schedule=pipelining_ops.PipelineSchedule.Sequential)
# Too many values:
with self.assertRaisesRegex(ValueError,
'Each stage must be mapped to an IPU'):
pipelining_ops.pipeline(
[stage1, stage2, stage3],
3,
inputs=[c],
infeed_queue=infeed_queue,
outfeed_queue=outfeed_queue,
device_mapping=list(range(4)),
pipeline_schedule=pipelining_ops.PipelineSchedule.Sequential)
# Not enough values:
with self.assertRaisesRegex(ValueError,
'Each stage must be mapped to an IPU'):
pipelining_ops.pipeline(
[stage1, stage2, stage3],
3,
inputs=[c],
infeed_queue=infeed_queue,
outfeed_queue=outfeed_queue,
device_mapping=tuple(range(1)),
pipeline_schedule=pipelining_ops.PipelineSchedule.Sequential)
def dataset_fn():
dataset = tu.create_single_increasing_dataset(7, shape=[1, 10])
return dataset.repeat().batch(4, drop_remainder=True)
def testPipelineInvalidDeviceMapping(self):
dataset = tu.create_single_increasing_dataset(5, shape=[4, 4, 2])
dataset = dataset.batch(batch_size=2, drop_remainder=True)
def dataset_parser(value):
a = value
b = (value + 10.) / 2.0
return {"a": a, "b": b}
dataset = dataset.map(dataset_parser)
infeed_queue = ipu_infeed_queue.IPUInfeedQueue(dataset, "__feed3")
outfeed_queue = ipu_outfeed_queue.IPUOutfeedQueue("__feed3")
def stage1(c, **kwargs):
with variable_scope.variable_scope("vs", use_resource=True):
y = layers.Conv2D(
2,
1,
use_bias=True,
kernel_initializer=init_ops.ones_initializer(),
name='conv1')(kwargs["a"])
return y + kwargs["b"], c
def stage2(x, c):
return math_ops.reduce_sum(x) + c
def stage3(x):
return x
with ops.device('cpu'):
c = array_ops.placeholder(np.float32, shape=[])
# Wrong type:
with self.assertRaisesRegex(
NotImplementedError,
'When using batch serialization, all the pipeline '
'stages need to be mapped to a single IPU.'):
pipelining_ops.pipeline(
[stage1, stage2, stage3],
3,
inputs=[c],
infeed_queue=infeed_queue,
outfeed_queue=outfeed_queue,
device_mapping=[0, 1, 0],
pipeline_schedule=pipelining_ops.PipelineSchedule.Sequential,
batch_serialization_iterations=4)
# Wrong type:
with self.assertRaisesRegex(
NotImplementedError,
'Batch serialization is only supported with the '
'`Sequential` schedule'):
pipelining_ops.pipeline(
[stage1, stage2, stage3],
3,
inputs=[c],
infeed_queue=infeed_queue,
outfeed_queue=outfeed_queue,
device_mapping=[0, 0, 0],
pipeline_schedule=pipelining_ops.PipelineSchedule.Grouped,
batch_serialization_iterations=4)
def dataset_fn(): dataset = tu.create_single_increasing_dataset(5, shape=[4, 4, 2]) dataset = dataset.batch(batch_size=2, drop_remainder=True) return dataset.map(dataset_parser)