def create_test_device(
numIpus: int = 1,
opts: Dict = None,
pattern: popart.SyncPattern = popart.SyncPattern.Full,
connectionType: popart.DeviceConnectionType = popart.
DeviceConnectionType.OnDemand,
selectionCriterion: popart.DeviceSelectionCriterion = popart.
DeviceSelectionCriterion.Random,
tilesPerIPU=None):
testDeviceType = os.environ.get("TEST_TARGET")
if opts:
if tilesPerIPU is not None and "tilesPerIPU" in opts.keys():
raise RuntimeError("Cannot set tilesPerIPU in 2 ways")
# NOTE: This function isn't symmetric with willow/include/popart/testdevice.hpp because it doesn't
# pass on the number of tiles for simulated devices (perhaps it should).
if tilesPerIPU is None and testDeviceType in ('Hw', 'IpuModel', 'Sim'):
tilesPerIPU = 4
if testDeviceType is None:
testDeviceType = "Cpu"
if testDeviceType == "Cpu":
device = popart.DeviceManager().createCpuDevice()
elif testDeviceType == "Sim":
if opts is None:
opts = {}
opts["numIPUs"] = numIpus
opts["tilesPerIPU"] = tilesPerIPU
device = popart.DeviceManager().createSimDevice(opts)
elif testDeviceType == "Hw":
dm = popart.DeviceManager()
# Keep trying to attach for 15 minutes before aborting
dm.setOnDemandAttachTimeout(900)
device = popart.DeviceManager().acquireAvailableDevice(
numIpus=numIpus,
tilesPerIpu=tilesPerIPU,
pattern=pattern,
connectionType=connectionType,
selectionCriterion=selectionCriterion)
assert device is not None
elif testDeviceType == "IpuModel":
if opts is None:
opts = {}
opts["numIPUs"] = numIpus
opts["tilesPerIPU"] = tilesPerIPU
device = popart.DeviceManager().createIpuModelDevice(opts)
else:
raise RuntimeError(f"Unknown device type {testDeviceType}")
if device is None:
return pytest.fail(
f"Tried to acquire device {testDeviceType} : {numIpus} IPUs, {tilesPerIPU} tiles,"
f" {pattern} pattern, {connectionType} connection, but none were availaible"
)
return device
def acquire_device(args, request_ipus):
if args.use_ipu_model:
device = popart.DeviceManager().createIpuModelDevice({"numIPUs": request_ipus})
else:
device = popart.DeviceManager().acquireAvailableDevice(request_ipus)
if device is None:
raise OSError("Failed to acquire IPU.")
logger.info(f"Acquired device: {device}")
return device
def init_deviceInfo(self, deviceType='ipu'):
"""compute how much ipu should used."""
if deviceType == 'ipu':
ipuCount = DeviceScope.IPUCount
ipu_num = pow(2, math.ceil(math.log2(ipuCount + 1)))
self.deviceInfo = popart.DeviceManager().acquireAvailableDevice(
ipu_num)
elif deviceType == 'cpu':
self.deviceInfo = popart.DeviceManager().createCpuDevice()
else:
raise RuntimeError('unknow device type')
def acquire_device(args, request_ipus):
if args.use_ipu_model:
model_opts = {"numIPUs": request_ipus}
if args.ipu_model_version is not None:
model_opts["ipuVersion"] = args.ipu_model_version
device = popart.DeviceManager().createIpuModelDevice(model_opts)
else:
sync_pattern = popart.SyncPattern.Full
device = popart.DeviceManager().acquireAvailableDevice(
request_ipus, 1216, pattern=sync_pattern)
if device is None:
raise OSError("Failed to acquire IPU.")
logger.info(f"Acquired device: {device}")
return device
def test_load_externally_saved_tensors():
"""
Test that initializer data can be saved in a separate file, and read into
the PopART IR in an InferenceSession (by observing an expected inference
result)
"""
builder = popart.Builder()
d1 = np.array([1, -1, 6]).astype(np.float32)
d2 = np.array([-8, 7, 4]).astype(np.float32)
i1 = builder.addInitializedInputTensor(d1)
i2 = builder.addInitializedInputTensor(d2)
o = builder.aiOnnx.add([i1, i2])
tmpdir = tempfile.mkdtemp()
tmpfile_tensors = os.path.join(tmpdir, "tensors.onnx")
tmpfile_model = os.path.join(tmpdir, "model.onnx")
builder.saveInitializersExternally([i1, i2], tmpfile_tensors)
builder.saveModelProto(tmpfile_model)
# Create builder from onnx model
builder = popart.Builder(tmpfile_model)
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
session = popart.InferenceSession(
fnModel=builder.getModelProto(),
dataFlow=dataFlow,
deviceInfo=popart.DeviceManager().createCpuDevice())
anchors = session.initAnchorArrays()
session.prepareDevice()
stepio = popart.PyStepIO({}, anchors)
session.run(stepio)
assert (np.array_equal(anchors[o], d1 + d2))
def get_session(fnModel, inputShapeInfo, dataFlow, torchWriter, patterns,
opts):
if len(torchWriter.outNames) > 1:
raise RuntimeError("Expecting single loss tensor")
# Append identity loss to output tensor
bder = popart.Builder(fnModel)
loss = bder.aiGraphcore.identityloss([torchWriter.outNames[0]])
# Reads ONNX model from file and creates backwards graph,
# performs Ir optimisations
session = popart.TrainingSession(
fnModel=bder.getModelProto(),
inputShapeInfo=inputShapeInfo,
dataFlow=dataFlow,
loss=loss,
optimizer=torchWriter.optimizer,
patterns=patterns,
userOptions=opts,
deviceInfo=popart.DeviceManager().createCpuDevice())
print("Setting device to IPU, and preparing it")
session.prepareDevice()
print("Writing weights to device")
session.weightsFromHost()
print("Writing Optimizer tensors to device, if there are any")
return session
def test_prune_all_error():
# Test that when all operations of a graph get pruned by the pruner a user-friendly exception is raised.
# Build an onnx model that performs an element-wise sprt operation
# on a 1D tensor containing 2 elements
builder = popart.Builder()
in0 = builder.addInputTensor("FLOAT", [2])
out = builder.aiOnnx.sqrt([in0])
# Construct a Session that can run this model. The ingredients:
# the onnx model
model = builder.getModelProto()
# the device to execute on (ipu, ipu model, or cpu)
deviceInfo = popart.DeviceManager().createCpuDevice()
# the anchors, or which model tensors to return to the host
anchors = [in0]
# how many times to execute the model before returning to the host
batchesPerStep = 1
# Exception should be thrown here.
with pytest.raises(popart.popart_exception) as e_info:
session = popart.InferenceSession(fnModel=model,
deviceInfo=deviceInfo,
dataFlow=popart.DataFlow(
batchesPerStep, anchors))
assert (e_info.value.args[0] ==
"All operations in the main graph were pruned, nothing to compute")
def get_device(conf):
""" Acquire IPU device """
device_manager = popart.DeviceManager()
if conf.simulation:
logger.info("Creating ipu sim")
ipu_options = {
"compileIPUCode": True,
'numIPUs': conf.num_ipus,
"tilesPerIPU": 1216
}
device = device_manager.createIpuModelDevice(ipu_options)
if device is None:
raise OSError("Failed to acquire IPU.")
else:
logger.info("Acquiring IPU")
if conf.select_ipu == 'AUTO':
device = device_manager.acquireAvailableDevice(conf.num_ipus)
else:
device = device_manager.acquireDeviceById(conf.select_ipu)
if device is None:
raise OSError("Failed to acquire IPU.")
else:
logger.info("Acquired IPU: {}".format(device))
return device
def test_set_and_get_ipu_model_version():
dm = popart.DeviceManager()
device = dm.createIpuModelDevice({'ipuVersion': 'ipu1'})
assert device.version == "ipu1"
device = dm.createIpuModelDevice({'ipuVersion': 'ipu2'})
assert device.version == "ipu2"
def test_binaryop_error():
M = 10
N = 10
K = 5
builder = popart.Builder(opsets={"ai.onnx": 9, "ai.graphcore": 1})
seq_info_A = popart.TensorInfo("FLOAT", [M, N])
A = builder.addInputTensor(seq_info_A, "A")
seq_info_B = popart.TensorInfo("FLOAT", [N, K])
B = builder.addInputTensor(seq_info_B, "B")
C = builder.aiOnnx.add([A, B], "C")
outputs = {C: popart.AnchorReturnType("ALL")}
for k in outputs.keys():
builder.addOutputTensor(k)
data_flow = popart.DataFlow(1, outputs)
model_opts = {"numIPUs": 1}
device = popart.DeviceManager().createIpuModelDevice(model_opts)
proto = builder.getModelProto()
with pytest.raises(popart.popart_exception) as e_info:
session = popart.InferenceSession(fnModel=proto,
deviceInfo=device,
dataFlow=data_flow)
assert ("np broadcasting failed on 'Op" in e_info.value.args[0])
assert ("ai.onnx.Add" in e_info.value.args[0])
assert ("frames [10, 10] and [10, 5] are not aligned"
in e_info.value.args[0])
def build_and_run_graph(input_data, run_on_ipu):
builder = popart.Builder()
input_len = len(input_data)
input_tensor = builder.addInputTensor(
popart.TensorInfo("FLOAT", [input_len]))
print("Shape of {}: {}".format(input_tensor,
builder.getTensorShape(input_tensor)))
output_tensor = builder.customOp(opName="Rsqrt",
opVersion=1,
domain="ai.graphcore",
inputs=[input_tensor],
attributes={})[0]
print("Inputs: {}".format(builder.getInputTensorIds()))
print("Outputs: {}".format(builder.getOutputTensorIds()))
print("Values: {}".format(builder.getValueTensorIds()))
print("Shape of {}: {}".format(output_tensor,
builder.getTensorShape(output_tensor)))
builder.addOutputTensor(output_tensor)
proto = builder.getModelProto()
anchors = {output_tensor: popart.AnchorReturnType("FINAL")}
dataFlow = popart.DataFlow(1, anchors)
if run_on_ipu:
device = popart.DeviceManager().acquireAvailableDevice(1)
print("IPU hardware device acquired")
else:
device = popart.DeviceManager().createIpuModelDevice({})
print("Running on IPU Model")
session = popart.InferenceSession(proto, dataFlow, device)
session.prepareDevice()
result = session.initAnchorArrays()
X = (np.array(input_data)).astype(np.float32)
print("X={}".format(X))
stepio = popart.PyStepIO({input_tensor: X}, result)
session.run(stepio)
return result
def acquire_device(args, request_ipus):
if args.use_ipu_model:
model_opts = {"numIPUs": request_ipus}
if args.ipu_model_version is not None:
model_opts["ipuVersion"] = args.device_version
device = popart.DeviceManager().createIpuModelDevice(model_opts)
else:
connection_type = popart.DeviceConnectionType.Always
if args.device_connection_type == "ondemand":
connection_type = popart.DeviceConnectionType.OnDemand
if args.execution_mode == "PHASED":
if args.phased_execution_type == "DUAL":
sync_pattern = popart.SyncPattern.ReplicaAndLadder
else:
sync_pattern = popart.SyncPattern.Full
elif args.execution_mode == "PIPELINE" and args.replication_factor <= 1:
sync_pattern = popart.SyncPattern.SinglePipeline
else:
sync_pattern = popart.SyncPattern.Full
manager = popart.DeviceManager()
manager.setOnDemandAttachTimeout(args.device_ondemand_timeout)
if args.device_connection_type == "offline":
opts = dict()
opts["numIPUs"] = request_ipus
opts["syncPattern"] = str(sync_pattern)
if args.device_tiles:
opts["tilesPerIPU"] = args.device_tiles
if args.device_version:
opts["ipuVersion"] = args.device_version
device = manager.createOfflineIPUDevice(opts)
else:
if args.device_id:
device = manager.acquireDeviceById(
args.device_id,
pattern=sync_pattern,
connectionType=connection_type)
else:
device = manager.acquireAvailableDevice(
request_ipus,
pattern=sync_pattern,
connectionType=connection_type)
if device is None:
raise OSError("Failed to acquire IPU.")
logger.info(f"Acquired device: {device}")
return device
def test_attached_state():
deviceManager = popart.DeviceManager()
device = deviceManager.acquireAvailableDevice(
1, connectionType=popart.DeviceConnectionType.OnDemand)
assert not device.isAttached
device.attach()
assert device.isAttached
device.detach()
assert not device.isAttached
def _init_session(self):
self._init_builder()
device = popart.DeviceManager().createOfflineIPUDevice({})
self.session = popart.InferenceSession(
fnModel=self.builder.getModelProto(),
dataFlow=popart.DataFlow(
1, {self.output: popart.AnchorReturnType("All")}),
deviceInfo=device)
def acquire_device(args, request_ipus):
if args.use_ipu_model:
model_opts = {"numIPUs": request_ipus}
if args.ipu_model_version is not None:
model_opts["ipuVersion"] = args.ipu_model_version
device = popart.DeviceManager().createIpuModelDevice(model_opts)
else:
if args.execution_mode == "PINGPONG":
sync_pattern = popart.SyncPattern.PingPong
elif args.execution_mode == "PIPELINE" and args.replication_factor <= 1:
sync_pattern = popart.SyncPattern.SinglePipeline
else:
sync_pattern = popart.SyncPattern.Full
device = popart.DeviceManager().acquireAvailableDevice(
request_ipus, pattern=sync_pattern)
if device is None:
raise OSError("Failed to acquire IPU.")
logger.info(f"Acquired device: {device}")
return device
def _test(transposedInput, transposedOutput):
builder = popart.Builder()
# Create a random constant and transpose it
np.random.seed(1)
input1 = builder.addInputTensor("INT32", [2, 2])
# Run a session to prove this
output1 = builder.aiOnnx.identity([input1])
builder.addOutputTensor(output1)
anchorConfig = {output1: popart.AnchorReturnType("ALL")}
dataFlow = popart.DataFlow(1, anchorConfig)
deviceConfig = {'numIPUs': 1}
dm = popart.DeviceManager()
device = dm.createIpuModelDevice(deviceConfig)
session = popart.InferenceSession(fnModel=builder.getModelProto(),
dataFlow=dataFlow,
deviceInfo=device)
# Compile graph and place weights onto it
session.prepareDevice()
session.weightsFromHost()
# Feed the session with a transposed (non-contiguous) tensor.
input1Value = np.random.randint(0, 100, size=(2, 2), dtype='int32')
if transposedInput:
input1Value = np.transpose(input1Value, [1, 0])
output1Value = np.random.randint(0, 100, size=(2, 2), dtype='int32')
if transposedOutput:
output1Value = np.transpose(output1Value, [1, 0])
with pytest.raises(
(Exception, RuntimeError, popart.popart_exception)) as e_info:
def input_callback(id, prefetch):
return input1Value
def input_complete_callback(id):
pass
def output_callback(id):
return output1Value
def output_complete_callback(id):
pass
stepio = popart.PyStepIOCallback(input_callback,
input_complete_callback,
output_callback,
output_complete_callback)
session.run(stepio)
assert "contiguous" in e_info.value.args[0]
def test_incomplete_grad():
# Reproducer for T37001, included as regression test. This test doesn't
# actually check any assertions, it just ensure that a code path that
# previously failed does not result in any exceptions.
#
# The problem originally revealed by this test was that an exception was
# thrown if for some inputs of a fwd subgraph the backwards pass creator was
# not able to create gradients for those inputs (for example for a seed
# input). This problem was fixed in the code base by allowing subgraph
# inputs in the fwd subgraph to not have an associated gradients outputs in
# the associated bwd subgraph.
def get_subgraph_builder(builder, weights, labels):
subgraph_builder = builder.createSubgraphBuilder()
subgraph_builder.addInputTensorFromParentGraph(weights)
input = subgraph_builder.addInputTensor(
popart.TensorInfo("FLOAT16", [4, 32, 1, 64]))
subgraph_builder.addInputTensorFromParentGraph(labels)
matmul_out = subgraph_builder.aiOnnx.matmul([input, weights])
log_probs = subgraph_builder.aiOnnx.logsoftmax([matmul_out], axis=3)
log_probs_compact = subgraph_builder.aiOnnx.gather([log_probs, labels],
axis=3)
subgraph_builder.addOutputTensor(log_probs_compact)
return subgraph_builder
builder = popart.Builder()
float16_input = builder.addInputTensor(
popart.TensorInfo("FLOAT16", [4, 32, 1, 64]), "float16_input")
int32_input = builder.addInputTensor(popart.TensorInfo("INT32", [4, 2]),
"int32_input")
weights = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16),
"weights")
fn = get_subgraph_builder(builder, weights, int32_input)
log_probs_compact = builder.aiGraphcore.call(
[weights, float16_input, int32_input], 1, fn)[0]
l1_loss = builder.aiGraphcore.l1loss([log_probs_compact], 1.0)
optimizer = popart.SGD({
"defaultLearningRate": (0.1, False),
"defaultWeightDecay": (0, True)
})
training_session = popart.TrainingSession(
builder.getModelProto(),
loss=l1_loss,
deviceInfo=popart.DeviceManager().createIpuModelDevice({}),
optimizer=optimizer,
dataFlow=popart.DataFlow(1, {}),
userOptions=popart.SessionOptions())
def create_test_device(numIpus: int = 1,
tilesPerIpu: int = 0,
opts: Dict = None,
pattern: popart.SyncPattern = popart.SyncPattern.Full,
connectionType: popart.DeviceConnectionType = popart.
DeviceConnectionType.Always):
testDeviceType = os.environ.get("TEST_TARGET")
# NOTE: This function isn't symmetric with willow/include/popart/testdevice.hpp because it doesn't
# pass on the number of tiles for simulated devices (perhaps it should).
if tilesPerIpu == 0 and testDeviceType == "IpuModel":
# We need a number of tiles for these device types.
tilesPerIpu = 1216
if opts is None:
opts = {}
opts["numIPUs"] = numIpus
opts["tilesPerIPU"] = tilesPerIpu
if testDeviceType is None:
testDeviceType = "Cpu"
if testDeviceType == "Cpu":
device = popart.DeviceManager().createCpuDevice()
elif testDeviceType == "Sim":
device = popart.DeviceManager().createSimDevice()
elif testDeviceType == "Hw":
device = popart.DeviceManager().acquireAvailableDevice(
numIpus=numIpus,
tilesPerIpu=tilesPerIpu,
pattern=pattern,
connectionType=connectionType)
elif testDeviceType == "IpuModel":
device = popart.DeviceManager().createIpuModelDevice(opts)
else:
raise RuntimeError(f"Unknown device type {testDeviceType}")
if device is None:
return pytest.fail(
f"Tried to acquire device {testDeviceType} : {numIpus} IPUs, {tilesPerIpu} tiles,"
f" {pattern} pattern, {connectionType} connection, but none were availaible"
)
return device
def test_aquire_device_by_id_2():
"""
Test that an error is thrown if trying to acquire device by id when it
has already been attached to
"""
deviceManager = popart.DeviceManager()
deviceId = 0
device = deviceManager.acquireDeviceById(deviceId)
with pytest.raises(popart.popart_exception,
match="Unable to acquire device with id") as e:
deviceManager.tryAcquireDeviceById(deviceId)
def test_aquire_device_by_id_3():
"""
Test that no error is thrown if trying to acquire device by id when it
has already been attached to, and that a null device is returned.
"""
deviceManager = popart.DeviceManager()
deviceId = 0
device0 = deviceManager.acquireDeviceById(deviceId)
assert device0 != None
device1 = deviceManager.acquireDeviceById(deviceId)
assert device1 == None
def test_aquire_device_by_id_1():
"""
Test that an error is thrown if trying to acquire device by id with
an invalid id
"""
deviceManager = popart.DeviceManager()
invalidId = 999
with pytest.raises(popart.poplar_exception,
match="No such device set id: '" + str(invalidId) +
"'") as e:
deviceManager.acquireDeviceById(invalidId)
def build_and_run_graph(input_data, alpha, run_on_ipu):
builder = popart.Builder()
input_len = len(input_data)
input_tensor = builder.addInputTensor(
popart.TensorInfo("FLOAT", [input_len]))
output_tensor = builder.customOp(opName="LeakyRelu",
opVersion=6,
domain="ai.onnx",
inputs=[input_tensor],
attributes={"alpha": alpha})[0]
builder.addOutputTensor(output_tensor)
proto = builder.getModelProto()
anchors = {output_tensor: popart.AnchorReturnType("FINAL")}
dataFeed = popart.DataFlow(1, anchors)
if run_on_ipu:
device = popart.DeviceManager().acquireAvailableDevice(1)
print("IPU hardware device acquired")
else:
device = popart.DeviceManager().createIpuModelDevice({})
print("Running on IPU Model")
print("alpha={}".format(alpha))
session = popart.InferenceSession(proto, dataFeed, device)
session.prepareDevice()
result = session.initAnchorArrays()
X = (np.array(input_data)).astype(np.float32)
print("X={}".format(X))
stepio = popart.PyStepIO({input_tensor: X}, result)
session.run(stepio)
return result
def run_with_input_of_type(dtype):
builder = popart.Builder()
in0 = builder.addInputTensor(popart.TensorInfo(dtype, [2]))
out = builder.aiOnnx.sqrt([in0])
opts = popart.SessionOptions()
opts.syntheticDataMode = popart.SyntheticDataMode.RandomNormal
session = popart.InferenceSession(
fnModel=builder.getModelProto(),
userOptions=opts,
deviceInfo=popart.DeviceManager().createCpuDevice(),
dataFlow=popart.DataFlow(1, [out]))
def get_device(sim=True):
# Select a device
deviceManager = popart.DeviceManager()
if sim:
options = {'compileIPUCode': True, 'numIPUs': 1, 'tilesPerIPU': 1216}
device = deviceManager.createIpuModelDevice(options)
else:
device = deviceManager.acquireAvailableDevice()
if device is None:
print('Failed to acquire IPU. Exiting.')
return None
return device
def test_aquire_device_by_id():
"""Test that aquiring by id works.
"""
deviceManager = popart.DeviceManager()
ipu_count = len(deviceManager.enumerateDevices(numIpus=1))
for id_ in range(ipu_count):
device = deviceManager.acquireDeviceById(
id=id_, pattern=popart.SyncPattern.Full)
# We can't be sure something else isn't using the IPUs...
if device is not None:
assert device.id == id_
device.detach()
def test_random_seed_setup():
ir = pir.Ir()
main = ir.main_graph()
with main:
seed_h2d = pir.h2d_stream(shape=(2, ),
dtype=dtypes.uint32,
name='seed_stream')
seed = ops.host_load(seed_h2d, 'seed')
x = pir.variable(0.0)
x = ops.dropout(x, seed + 1, p=0.1)
y = ops.dropout(x, seed + 2, p=0.7)
y_d2h = pir.d2h_stream(y.shape, y.dtype, name="y_stream")
ops.host_store(y_d2h, y)
replicas = 4
parent_seed = 1984
seed_tensors = pir.create_seeds(parent_seed, replicas=replicas)
## Run the program
ir = ir._pb_ir # Internal ir
y_id = y_d2h.tensor_id()
dataFlow = popart.DataFlow(
batchesPerStep=1, anchorTensors={y_id: popart.AnchorReturnType("All")})
ir.setDataFlow(dataFlow)
opts = ir.getSessionOptions()
opts.useHostCopyOps = True
opts.enableExplicitMainLoops = True
opts.aliasZeroCopy = True
opts.explicitRecomputation = True
opts.enableReplicatedGraphs = True
opts.replicatedGraphCount = replicas
ir.updateVertices()
device = popart.DeviceManager().createIpuModelDevice({"numIPUs": replicas})
session = popart.InferenceSession.fromIr(ir=ir, deviceInfo=device)
session.prepareDevice()
# Create buffers for anchors
anchors = session.initAnchorArrays()
# Run the model
stepio = popart.PyStepIO(inputs={seed_h2d.tensor_id(): seed_tensors},
outputs=anchors)
session.weightsFromHost()
session.run(stepio)
def test_compileAndExport_model(self):
self._init_builder()
device = popart.DeviceManager().createIpuModelDevice({})
session = popart.InferenceSession(
fnModel=self.builder.getModelProto(),
dataFlow=popart.DataFlow(
1, {self.output: popart.AnchorReturnType("All")}),
deviceInfo=device)
with tempfile.TemporaryDirectory() as tmpdirname:
with pytest.raises(popart.popart_exception) as e:
session.compileAndExport(tmpdirname)
assert "Offline compilation is not supported" in str(e.value)
def test_default_connection_type_1():
"""
Test that error is thrown if try to acquire more devices than are
available
"""
deviceManager = popart.DeviceManager()
ipus = 256 # More IPUs than are available
with pytest.raises(
popart.popart_exception,
match=
"Failed to acquire device. Ensure that there are sufficient IPUs available"
) as e:
deviceManager.tryAcquireAvailableDevice(ipus)
def test_postnrepl_overzealous_elimination():
# Reproducer for T36270, included as regression test. This test doesn't
# actually do any assertions, it just checks that a code path that
# previously failed does not result in any exceptions.
#
# The bug was that the PostNRepl pattern removed the gradient sum op that
# produces Gradient_<in0> (which has 1 input) in the backwards subgraph, also
# rewriting the subgraph itself to use the input to the gradient sum op
# instead, as it's identical. However, the tensor produced by the op is a
# graph output that is used by a call op in the main graph. The pattern did
# not adjust this CallOp or the subgraph's output tensors and so the CallOp
# in the main graph fails because it's using a tensor that no longer exists.
def get_subgraph_builder(b, w):
builder = b.createSubgraphBuilder()
builder.addInputTensorFromParentGraph(w)
in0 = builder.addInputTensor(
popart.TensorInfo("FLOAT16", [4, 32, 16, 64]))
x = builder.aiOnnx.matmul([in0, w])
builder.addOutputTensor(x)
return builder
# building model and dataflow
builder = popart.Builder()
in0 = builder.addInputTensor(popart.TensorInfo('FLOAT16', [4, 32, 1, 64]),
"in0")
w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16),
"weights")
fn = get_subgraph_builder(builder, w)
x = builder.aiGraphcore.call([w, in0], 1, fn)[0]
l1_loss = builder.aiGraphcore.l1loss([x], 1.0)
optimizer = popart.SGD({
"defaultLearningRate": (0.1, False),
"defaultWeightDecay": (0, True)
})
device = popart.DeviceManager().createIpuModelDevice({})
# create training session
popart.TrainingSession(fnModel=builder.getModelProto(),
loss=l1_loss,
deviceInfo=device,
optimizer=optimizer,
dataFlow=popart.DataFlow(1, {}),
userOptions=popart.SessionOptions())
def test_default_connection_type_2():
"""
Test that error is thrown if try to acquire a single IPU when all have
already been attached to
"""
deviceManager = popart.DeviceManager()
availableIpus = len(deviceManager.enumerateDevices())
d0 = deviceManager.acquireAvailableDevice(availableIpus)
with pytest.raises(
popart.popart_exception,
match=
"Failed to acquire device. Ensure that there are sufficient IPUs available"
) as e:
deviceManager.tryAcquireAvailableDevice(1)
