def test_ipu_copy_aca1():
popart.getLogger().setLevel("TRACE")
builder = popart.Builder()
i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1]))
i2 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1]))
o1 = builder.aiOnnx.add([i1, i2])
o2 = builder.aiOnnx.add([i1, i2])
o = builder.aiOnnx.add([o1, o2])
builder.addOutputTensor(o)
builder.virtualGraph(o1, 0)
builder.virtualGraph(o2, 0)
builder.virtualGraph(o, 10) # << Invalid virtual graph
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
opts = popart.SessionOptions()
opts.virtualGraphMode = popart.VirtualGraphMode.Manual
s = popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
deviceInfo=tu.create_test_device(numIpus=3))
with pytest.raises(popart.popart_exception) as e_info:
s.prepareDevice()
assert (("inputs=[{}, {}], outputs=[{}]) " +
"has been assigned to an invalid virtual graph 10").format(
o1 + "_c10", o2 + "_c10", o) in e_info.value.args[0])
def test_engine_options_passed_to_engine(tmpdir):
popart.getLogger().setLevel("DEBUG")
builder = popart.Builder()
shape = popart.TensorInfo("FLOAT", [1, 2, 32, 32])
i1 = builder.addInputTensor(shape)
i2 = builder.addInputTensor(shape)
o = builder.aiOnnx.add([i1, i2])
builder.addOutputTensor(o)
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
opts = popart.SessionOptions()
opts.engineOptions = {'option': 'value'}
session = popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
deviceInfo=tu.create_test_device())
session.initAnchorArrays()
with pytest.raises(popart.poplar_exception) as e_info:
session.prepareDevice()
assert (e_info.value.args[0].endswith("Unrecognised option 'option'"))
def test_ipu_copy_bca5():
popart.getLogger().setLevel("TRACE")
builder = popart.Builder()
constData = np.random.rand(2, 2).astype(np.float32)
c1 = builder.aiOnnx.constant(constData, "constShapeData")
i2 = builder.addInputTensor(popart.TensorInfo("FLOAT", [2, 2]))
o1 = builder.aiOnnx.add([c1, i2])
o2 = builder.aiOnnx.add([c1, i2])
t1 = builder.aiOnnx.transpose([c1], [])
o3 = builder.aiOnnx.add([o1, o2])
o = builder.aiOnnx.add([o3, t1])
builder.addOutputTensor(o)
builder.virtualGraph(o1, 0)
builder.virtualGraph(o2, 2)
builder.virtualGraph(t1, 2)
builder.virtualGraph(o3, 1)
builder.virtualGraph(o, 1)
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
opts = popart.SessionOptions()
opts.virtualGraphMode = popart.VirtualGraphMode.Manual
s = popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
deviceInfo=tu.create_test_device(numIpus=3))
s.prepareDevice()
def test_virtual_graph_multi_output(container):
popart.getLogger().setLevel("TRACE")
builder = popart.Builder()
i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [4, 1]))
o1, o2, o3, o4 = builder.aiOnnx.split([i1], 4)
o5 = builder.aiOnnx.add([o1, o2])
o6 = builder.aiOnnx.add([o3, o4])
o = builder.aiOnnx.add([o5, o6])
builder.addOutputTensor(o)
builder.virtualGraph(container([o1, o2, o3, o4]), 1)
builder.virtualGraph(o5, 0)
builder.virtualGraph(o6, 0)
builder.virtualGraph(o, 0)
# Check the virtual graph was set correctly
assert builder.getVirtualGraph(container([o1, o2, o3, o4])) == 1
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
opts = popart.SessionOptions()
opts.virtualGraphMode = popart.VirtualGraphMode.Manual
s = popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
deviceInfo=tu.create_test_device(numIpus=2))
s.prepareDevice()
def test_copy_to_op_with_duplicate_inputs():
popart.getLogger().setLevel("TRACE")
builder = popart.Builder()
i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1]))
o1 = builder.aiOnnx.add([i1, i1])
builder.virtualGraph(o1, 0)
o2 = builder.aiOnnx.mul([i1, i1])
builder.virtualGraph(o2, 1)
o3 = builder.aiOnnx.add([o1, o2])
builder.virtualGraph(o3, 1)
o = o3
builder.addOutputTensor(o)
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
opts = popart.SessionOptions()
opts.virtualGraphMode = popart.VirtualGraphMode.Manual
s = popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
deviceInfo=tu.create_test_device(numIpus=3))
s.prepareDevice()
def test_mixed_virtual_graph():
popart.getLogger().setLevel("TRACE")
builder = popart.Builder()
i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1]))
i2 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1]))
o1 = builder.aiOnnx.add([i1, i2])
o2 = builder.aiOnnx.add([i1, i2])
o = builder.aiOnnx.add([o1, o2])
builder.addOutputTensor(o)
builder.virtualGraph(o1, 0)
builder.virtualGraph(o2, 1)
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
opts = popart.SessionOptions()
opts.virtualGraphMode = popart.VirtualGraphMode.Manual
with pytest.raises(popart.popart_exception) as e_info:
popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
deviceInfo=tu.create_test_device())
assert (e_info.value.args[0].startswith("Either all"))
def test_virtual_graph():
popart.getLogger().setLevel("TRACE")
builder = popart.Builder()
i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1]))
i2 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1]))
i3 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1]))
i4 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1]))
o1 = builder.aiOnnx.add([i1, i2])
o2 = builder.aiOnnx.add([i3, i4])
o = builder.aiOnnx.add([o1, o2])
builder.addOutputTensor(o)
builder.virtualGraph(o1, 1)
builder.virtualGraph(o2, 1)
builder.virtualGraph(o, 1)
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
opts = popart.SessionOptions()
opts.virtualGraphMode = popart.VirtualGraphMode.Manual
s = popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
deviceInfo=tu.create_test_device(numIpus=2))
s.prepareDevice()
pass
def test_auto_virtual_graph_not_enough_splits():
ipus = 4
popart.getLogger().setLevel("TRACE")
builder = popart.Builder()
input_shape = [1, 64]
input = builder.addInputTensor(popart.TensorInfo("FLOAT16", input_shape))
x = input
for i in range(2):
w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16))
x = builder.aiOnnx.matmul([x, w])
output = x
builder.addOutputTensor(output)
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {output: popart.AnchorReturnType("Final")})
opts = popart.SessionOptions()
opts.virtualGraphMode = popart.VirtualGraphMode.Auto
device = tu.create_test_device(numIpus=ipus)
with pytest.raises(popart.popart_exception) as e_info:
popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
deviceInfo=device)
assert (e_info.value.args[0].startswith(
"[AutoVirtualGraph] Couldn't find enough splits"))
def test_auto_virtual_graph_inf_many():
ipus = 4
popart.getLogger().setLevel("TRACE")
builder = popart.Builder()
input_shape = [1, 64]
input = builder.addInputTensor(popart.TensorInfo("FLOAT16", input_shape))
x = input
for i in range(16):
w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16))
x = builder.aiOnnx.matmul([x, w])
output = x
builder.addOutputTensor(output)
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {output: popart.AnchorReturnType("Final")})
opts = popart.SessionOptions()
opts.virtualGraphMode = popart.VirtualGraphMode.Auto
device = tu.create_test_device(numIpus=ipus)
popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
deviceInfo=device)
def test_set_random_seed_error():
popart.getLogger().setLevel("TRACE")
builder = popart.Builder()
i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [10]))
[o] = builder.aiOnnx.dropout([i1], num_outputs=1, ratio=0.3)
loss = builder.aiGraphcore.identityloss([o])
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
s = popart.TrainingSession(fnModel=proto,
dataFlow=dataFlow,
optimizer=popart.ConstSGD(0.1),
loss=loss,
userOptions=popart.SessionOptions(),
deviceInfo=tu.create_test_device(numIpus=2))
with pytest.raises(popart.popart_exception) as e_info:
s.setRandomSeed(0)
msg = e_info.value.args[0]
assert msg == ("Devicex::prepare() must be called before "
"Devicex::setRandomSeedFromHost(uint64_t) is called.")
def test_model_caching(tmp_path, capfd):
# Check the log output to see if an engine was compiled,
# or if a cached engine was used.
def loaded_saved_executable():
_, stderr = capfd.readouterr()
startedEngineCompilation = False
loadedPoplarExecutable = False
for line in stderr.splitlines():
if 'Starting compilation' in line:
startedEngineCompilation = True
elif 'Loading serialized PopART executable' in line:
loadedPoplarExecutable = True
assert startedEngineCompilation != loadedPoplarExecutable
return startedEngineCompilation
popart.getLogger().setLevel('DEBUG')
cache_path = str(tmp_path / 'model_caching')
result0 = build_and_run(cache_path)
assert loaded_saved_executable() is True
result1 = build_and_run(cache_path)
assert loaded_saved_executable() is False
assert result0 == result1
shutil.rmtree(cache_path)
build_and_run(cache_path)
assert loaded_saved_executable() is True
def test_average_pool_with_count_include_pad(op_tester):
popart.getLogger().setLevel("TRACE")
builder = popart.Builder()
i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1, 1, 14, 14]))
o = builder.aiOnnx.averagepool([i1],
kernel_shape=[3, 3],
count_include_pad=1,
pads=[0, 0, 0, 0],
strides=[2, 2])
builder.addOutputTensor(o)
builder.removeNodeAttribute("count_include_pad", set([o]))
builder.addNodeAttribute("count_include_pad", 1, set([o]))
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
optimizer = popart.ConstSGD(0.01)
loss = builder.aiGraphcore.l1loss([o], 0.1)
proto = builder.getModelProto()
opts = popart.SessionOptions()
with pytest.raises(popart.popart_exception) as e_info:
popart.TrainingSession(fnModel=proto,
dataFlow=dataFlow,
loss=loss,
optimizer=optimizer,
userOptions=opts,
deviceInfo=tu.create_test_device())
assert (e_info.value.args[0].startswith(
"`count_include_pad` is not supported"))
def __init__(self,
dataset,
batch_size=1,
shuffle=False,
sampler=None,
batch_sampler=None,
num_workers=0,
collate_fn=default_collate,
drop_last=False,
tensor_type=None,
timeout=0,
worker_init_fn=None,
log_statisics=False):
self.dataset = dataset
self.batch_size = batch_size
self.num_workers = num_workers
self.collate_fn = partial(collate_fn, tensor_type=tensor_type)
self.drop_last = drop_last
self.timeout = timeout
self.worker_init_fn = worker_init_fn
self.log_statisics = log_statisics
popart.getLogger().info(
"DataLoader created batchsize:{} num_workers:{} "
"shuffle:{} tensor_type:{}".format(batch_size, num_workers,
shuffle, tensor_type))
if timeout < 0:
raise ValueError('timeout option should be non-negative')
if batch_sampler is not None:
if batch_size > 1 or shuffle or sampler is not None or drop_last:
raise ValueError('batch_sampler option is mutually exclusive '
'with batch_size, shuffle, sampler, and '
'drop_last')
self.batch_size = None
self.drop_last = None
if sampler is not None and shuffle:
raise ValueError('sampler option is mutually exclusive with '
'shuffle')
if self.num_workers < 0:
raise ValueError('num_workers option cannot be negative; '
'use num_workers=0 to disable multiprocessing.')
if batch_sampler is None:
if sampler is None:
if shuffle:
sampler = torch.utils.data.RandomSampler(dataset)
else:
sampler = torch.utils.data.SequentialSampler(dataset)
batch_sampler = torch.utils.data.BatchSampler(
sampler, batch_size, drop_last)
self.sampler = sampler
self.batch_sampler = batch_sampler
self.dataloaderiter = None
def check_loglevel(level):
popart.getLogger().setLevel(level)
builder = popart.Builder()
i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1, 2, 3]))
o = builder.aiOnnx.abs([i1])
_, err = capfd.readouterr()
return err
def test_constants_preserved():
popart.getLogger().setLevel("TRACE")
# Check that `session.modelToHost` can be called when using a
# model with a constant node, without throwing an exceptions
builder = popart.Builder()
i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [2, 2]))
i2 = builder.addInputTensor(popart.TensorInfo("FLOAT", [2, 2]))
c = builder.aiOnnx.constant(np.array([[1, 2], [3, 4]], dtype=np.float32))
o1 = builder.aiOnnx.add([i1, i2])
o2 = builder.aiOnnx.add([o1, c])
loss = builder.aiGraphcore.identityloss([o2])
proto = builder.getModelProto()
anchors = {o2: popart.AnchorReturnType("All")}
dataFlow = popart.DataFlow(1, anchors)
optimizer = popart.ConstSGD(0.01)
opts = popart.SessionOptions()
session = popart.TrainingSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
loss=loss,
optimizer=optimizer,
deviceInfo=tu.create_test_device())
anchorArrays = session.initAnchorArrays()
session.prepareDevice()
inputs = {
i1: np.array([[2, 2], [2, 2]]).astype(np.float32),
i2: np.array([[4, 4], [4, 4]]).astype(np.float32),
}
pystepio = popart.PyStepIO(inputs, anchorArrays)
session.run(pystepio)
session.modelToHost('session_proto.onnx')
# models should be the same after training
# as there are no trainable parameters
with open('session_proto.onnx', 'rb') as f:
session_proto = f.read()
assert proto == session_proto
# confirm that the output is correct. See T6186, which this tests
assert (np.sum(np.abs(anchorArrays[o2] - np.array([[7, 8], [9, 10]]))) <
1e-8)
def test_sgd_with_float16_model():
popart.getLogger().setLevel("TRACE")
input1 = np.zeros((2, 2, 4, 4), dtype=np.float16)
input2 = np.zeros((2, 2, 3, 3), dtype=np.float16)
input3 = np.zeros((2, 2, 3, 3), dtype=np.float16)
builder = popart.Builder()
inid1 = builder.addInputTensor(popart.TensorInfo(input1))
inid2 = builder.addInitializedInputTensor(input2)
inid3 = builder.addInitializedInputTensor(input2)
c1 = builder.aiOnnx.conv([inid1, inid2],
dilations=[1, 1],
pads=[1, 1, 1, 1],
strides=[1, 1])
c2 = builder.aiOnnx.conv([c1, inid3],
dilations=[1, 1],
pads=[1, 1, 1, 1],
strides=[1, 1])
# Reduce to scalar
out = builder.aiGraphcore.identityloss([c2])
proto = builder.getModelProto()
optimizer = popart.SGD({
"defaultLearningRate": (0.1, False),
"defaultWeightDecay": (0.1, False),
"lossScaling": (1000, False)
})
anchorNames = {
popart.reservedGradientPrefix() + inid1:
popart.AnchorReturnType("All"),
}
opts = popart.SessionOptions()
session = popart.TrainingSession(
fnModel=proto,
dataFlow=popart.DataFlow(1, anchorNames),
loss=out,
optimizer=optimizer,
deviceInfo=tu.create_test_device(opts={"compileIPUCode": False}))
session.prepareDevice()
session.weightsFromHost()
anchorArrays = session.initAnchorArrays()
stepio = popart.PyStepIO({inid1: input1}, anchorArrays)
session.run(stepio)
def prepare(
cls,
model, # type: ModelProto
device='IPU', # type: Text
**kwargs # type: Any
): # type: (...) -> Optional[onnx.backend.base.BackendRep]
super(IpuBackend, cls).prepare(model, device, **kwargs)
# test shape inference
model = onnx.shape_inference.infer_shapes(model)
value_infos = {
vi.name: vi
for vi in itertools.chain(model.graph.value_info,
model.graph.output)
}
# if do_enforce_test_coverage_whitelist(model):
# for node in model.graph.node:
# for i, output in enumerate(node.output):
# if node.op_type == 'Dropout' and i != 0:
# continue
# assert output in value_infos
# tt = value_infos[output].type.tensor_type
# assert tt.elem_type != TensorProto.UNDEFINED
# for dim in tt.shape.dim:
# assert dim.WhichOneof('value') == 'dim_value'
popart.getLogger().setLevel("DEBUG")
opts = popart.SessionOptions()
anchors = {}
for output in model.graph.output:
anchors[output.name] = popart.AnchorReturnType("All")
session = popart.InferenceSession(
fnModel=model.SerializeToString(),
dataFlow=popart.DataFlow(1, anchors),
deviceInfo=popart.DeviceManager().createCpuDevice(),
# deviceInfo=popart.DeviceManager().createIpuModelDevice({}),
userOptions=opts)
session.prepareDevice()
context = Context(session, model)
return context
def test_cache_environment_variable(tmp_path, capfd):
"""
Test chaching enabled via env POPART_CACHE_DIR
"""
popart.getLogger().setLevel('DEBUG')
os.environ['POPART_CACHE_DIR'] = str(tmp_path / 'saved_graph')
opts = popart.SessionOptions()
run_session(2, opts)
assert loaded_saved_executable(capfd) is False
# Check engine caching works for two identical sessions.
run_session(2, opts)
assert loaded_saved_executable(capfd) is True
del os.environ['POPART_CACHE_DIR']
def test_auto_virtual_graph_subgraphs_2():
ipus = 2
popart.getLogger().setLevel("TRACE")
builder = popart.Builder()
input_shape = [1, 64]
input1 = builder.addInputTensor(popart.TensorInfo("FLOAT16", input_shape))
input2 = builder.addInputTensor(popart.TensorInfo("FLOAT16", input_shape))
# Subgraph 0
w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16))
x0 = builder.aiOnnx.matmul([input1, w])
w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16))
x0 = builder.aiOnnx.matmul([x0, w])
# Subgraph 1
w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16))
x1 = builder.aiOnnx.matmul([input2, w])
# Subgraph 2
x2 = builder.aiOnnx.add([x0, x1])
w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16))
x2 = builder.aiOnnx.matmul([x2, w])
output = x2
builder.addOutputTensor(output)
# Desired split is:
# ipu1: 0. ipu2: 1,2
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {output: popart.AnchorReturnType("Final")})
opts = popart.SessionOptions()
opts.virtualGraphMode = popart.VirtualGraphMode.Auto
device = tu.create_test_device(numIpus=ipus)
popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
deviceInfo=device)
def test_auto_virtual_graph_subgraphs_1():
ipus = 1
popart.getLogger().setLevel("TRACE")
builder = popart.Builder()
input_shape = [1, 64]
input1 = builder.addInputTensor(popart.TensorInfo("FLOAT16", input_shape))
input2 = builder.addInputTensor(popart.TensorInfo("FLOAT16", input_shape))
# Subgraph 0
w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16))
x0 = builder.aiOnnx.matmul([input1, w])
w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16))
x0 = builder.aiOnnx.matmul([x0, w])
# Subgraph 1
w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16))
x1 = builder.aiOnnx.matmul([input2, w])
output = x1
builder.addOutputTensor(output)
# Only possible split is:
# ipu1: 0, 1
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {output: popart.AnchorReturnType("Final")})
opts = popart.SessionOptions()
opts.virtualGraphMode = popart.VirtualGraphMode.Auto
device = tu.create_test_device(numIpus=ipus)
session = popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
deviceInfo=device)
ir = json.loads(session._serializeIr(popart.IrSerializationFormat.JSON))
for op in ir["maingraph"]:
print(op)
assert (int(op["attributes"]["__ipu_number"]) == 0)
def reset(self):
popart.getLogger().debug("Resetting the dataloaderiterator")
# Drain the workers
self.flush_data_queue = True
while self.batches_outstanding > 0:
self.__next__()
self.flush_data_queue = False
# Reset the sample iterator
self.send_idx = 0
self.rcvd_idx = 0
self.sample_iter = iter(self.batch_sampler)
# prime the prefetch loop
for _ in range(2 * self.num_workers):
self._put_indices()
def trainSession(anchors, optimizer, stepSize):
popart.getLogger().setLevel("TRACE")
builder = popart.Builder()
dataShape = popart.TensorInfo("FLOAT", [1, 2, 4, 4])
i1 = builder.addInputTensor(dataShape)
filtInit = np.ones([2, 2, 3, 3], dtype=np.float32)
i2 = builder.addInitializedInputTensor(filtInit)
c1 = builder.aiOnnx.conv([i1, i2],
dilations=[1, 1],
pads=[1, 1, 1, 1],
strides=[1, 1])
c2 = builder.aiOnnx.conv([c1, i2],
dilations=[1, 1],
pads=[1, 1, 1, 1],
strides=[1, 1])
o = builder.aiGraphcore.l1loss([c2], 0.1)
proto = builder.getModelProto()
opts = popart.SessionOptions()
session = popart.TrainingSession(
fnModel=proto,
dataFlow=popart.DataFlow(stepSize, anchors),
loss=o,
optimizer=optimizer,
deviceInfo=tu.create_test_device(opts={"compileIPUCode": False}))
session.prepareDevice()
session.weightsFromHost()
# add step dimension to infeed
infeedShape = dataShape.shape()
infeedShape.insert(0, stepSize)
data = np.ones(infeedShape, dtype=np.float32)
inputs = {i1: data}
return session, inputs
def run(torchWriter,
patterns,
outputdir,
cifarInIndices,
device,
device_hw_id,
mode="train",
syntheticData=False,
transformations=[],
epochs=4,
printAnchorArrays=False):
popart.getLogger().setLevel("TRACE")
popart.getLogger("session").setLevel("WARN")
if outputdir is None:
with TemporaryDirectory() as outputdir:
return _run_impl(torchWriter,
patterns,
outputdir,
cifarInIndices,
device,
device_hw_id,
mode,
syntheticData,
transformations,
epochs,
printAnchorArrays=printAnchorArrays)
else:
if not os.path.exists(outputdir):
os.mkdir(outputdir)
return _run_impl(torchWriter,
patterns,
outputdir,
cifarInIndices,
device,
device_hw_id,
mode,
syntheticData,
transformations,
epochs,
printAnchorArrays=printAnchorArrays)
def test_simple_cache_hit(tmp_path, capfd):
"""
Test:
1. That engine caching works for two identical sessions
2. That the cached engine isn't loaded for a different session
"""
popart.getLogger().setLevel('DEBUG')
opts = popart.SessionOptions()
opts.enableEngineCaching = True
opts.cachePath = str(tmp_path / 'saved_graph')
run_session(2, opts)
assert loaded_saved_executable(capfd) is False
# Check engine caching works for two identical sessions.
run_session(2, opts)
assert loaded_saved_executable(capfd) is True
# Check the cached engine isn't loaded for a different session.
run_session(70, opts)
assert loaded_saved_executable(capfd) is False
def test_no_prepare_device():
popart.getLogger().setLevel("TRACE")
# Check that `session.modelToHost` can be called when using a
# model with a constant node, without throwing an exceptions
builder = popart.Builder()
i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [2, 2]))
i2 = builder.addInputTensor(popart.TensorInfo("FLOAT", [2, 2]))
c = builder.aiOnnx.constant(np.array([[1, 2], [3, 4]], dtype=np.float32))
o1 = builder.aiOnnx.add([i1, i2])
o2 = builder.aiOnnx.add([o1, c])
loss = builder.aiGraphcore.identityloss([o2])
proto = builder.getModelProto()
anchors = {o2: popart.AnchorReturnType("All")}
dataFlow = popart.DataFlow(1, anchors)
optimizer = popart.ConstSGD(0.01)
opts = popart.SessionOptions()
session = popart.TrainingSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
loss=loss,
optimizer=optimizer,
deviceInfo=tu.create_test_device())
# No session.prepareDevice()
with pytest.raises(popart.popart_exception) as e_info:
session.modelToHost('session_proto.onnx')
assert (e_info.value.args[0].startswith(
"Devicex::prepare() must be called before Devicex::weightsToHost"))
def test_no_virtual_graph():
popart.getLogger().setLevel("TRACE")
builder = popart.Builder()
i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1]))
i2 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1]))
o1 = builder.aiOnnx.add([i1, i2])
o2 = builder.aiOnnx.add([i1, i2])
o = builder.aiOnnx.add([o1, o2])
builder.addOutputTensor(o)
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
opts = popart.SessionOptions()
popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
deviceInfo=tu.create_test_device())
def test_verify_synthetic_inputs(capfd, inputType):
"""
For each synthetic data mode:
1. Get a session that prints the input tensor value to stderr
2. Capture the tensor data from stderr
3. Verify that the data is as expected for that synthetic data mode
"""
# Hopefully this is large enough to achieve desired tolerance for mean/std,
# even for ints.
d_shape = [4000]
# Test depends on logging output. Silence the logging from PopART
popart.getLogger().setLevel("OFF")
## A) Expect input is all zeros
run_pt_session(popart.SyntheticDataMode.Zeros,
inputType=inputType,
d_shape=d_shape)
_, err0 = capfd.readouterr()
zeroData = numpy_array_from_printtensor_string(err0)
assert np.all(zeroData == 0)
## B) Expect input is random normal, T~N(0,1)
if inputType == _UINT8:
# Casting normal data to unsigned results in non-normal data.
return
run_pt_session(popart.SyntheticDataMode.RandomNormal,
inputType=inputType,
d_shape=d_shape)
_, err1 = capfd.readouterr()
rnData = numpy_array_from_printtensor_string(err1)
assert np.all(rnData == 0) == False
assert np.isclose(np.mean(rnData), 0, atol=0.02)
assert np.isclose(np.std(rnData), 1, atol=0.1)
def __next__(self):
if self.num_workers == 0: # same-process loading
indices = next(self.sample_iter) # may raise StopIteration
if self.log_statisics:
t = time.time()
batch = self.collate_fn([self.dataset[i] for i in indices])
self.processing_times.append(time.time() - t)
if len(self.processing_times) > 8:
popart.getLogger().info(
"DataLoader processing:{0:6.4f}".format(
sum(self.processing_times) /
len(self.processing_times)))
self.processing_times.clear()
else:
batch = self.collate_fn([self.dataset[i] for i in indices])
return [x.numpy() for x in batch]
# check if the next sample has already been generated
if self.rcvd_idx in self.reorder_dict:
batch = self.reorder_dict.pop(self.rcvd_idx)
return self._process_next_batch(batch)
# raise the stop iteration execption when we have recevied all
# batches in the data set
if self.batches_outstanding == 0:
raise StopIteration
while True:
assert (not self.shutdown and self.batches_outstanding > 0)
idx, batch = self._get_batch()
self.batches_outstanding -= 1
if idx != self.rcvd_idx:
# store out-of-order samples
self.reorder_dict[idx] = batch
continue
return self._process_next_batch(batch)
def test_auto_virtual_graph_train():
ipus = 2
popart.getLogger().setLevel("TRACE")
builder = popart.Builder()
input_shape = [1, 64]
input = builder.addInputTensor(popart.TensorInfo("FLOAT16", input_shape))
x = input
for i in range(ipus):
w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16))
x = builder.aiOnnx.matmul([x, w])
output = x
builder.addOutputTensor(output)
loss = builder.aiGraphcore.identityloss([output])
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {loss: popart.AnchorReturnType("Final")})
opts = popart.SessionOptions()
opts.virtualGraphMode = popart.VirtualGraphMode.Auto
device = tu.create_test_device(numIpus=ipus)
popart.TrainingSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
loss=loss,
optimizer=popart.SGD(
{"defaultLearningRate": (0.01, True)}),
deviceInfo=device)
def test_verify_synthetic_inputs(capfd):
"""
For each synthetic data mode:
1. Get a session that prints the input tensor value to stderr
2. Capture the tensor data from stderr
3. Verify that the data is as expected for that synthetic data mode
"""
# Test depends on logging output. Silence the logging from PopART
popart.getLogger().setLevel("OFF")
## A) Expect input is all zeros
run_pt_session(popart.SyntheticDataMode.Zeros)
_, err0 = capfd.readouterr()
zeroData = numpy_array_from_printtensor_string(err0)
assert np.all(zeroData == 0)
## B) Expect input is random normal, T~N(0,1)
run_pt_session(popart.SyntheticDataMode.RandomNormal)
_, err1 = capfd.readouterr()
rnData = numpy_array_from_printtensor_string(err1)
assert np.all(rnData == 0) == False
assert np.isclose(np.mean(rnData), 0, atol=0.02)
assert np.isclose(np.std(rnData), 1, atol=0.1)
