def test_dropout_training7():
dsize = 100
ratio = 0.2
builder = popart.Builder()
ip = builder.addInputTensor(popart.TensorInfo("FLOAT", [dsize]))
d__ip = popart.reservedGradientPrefix() + ip
[d1] = builder.aiOnnx.dropout([ip], num_outputs=1, ratio=ratio)
[d2] = builder.aiOnnx.dropout([ip], num_outputs=1, ratio=ratio)
out = builder.aiOnnx.add([d1, d2])
out = builder.aiGraphcore.identityloss([out])
builder.addOutputTensor(out)
if tu.ipu_available():
device = tu.create_test_device()
else:
pytest.skip("Test needs to run on IPU, but none are available")
session, anchors = get_session(anchorIds=[d1, d2],
proto=builder.getModelProto(),
device=device,
loss=out)
# Same data for each batch
ip_data = np.random.random_sample(dsize).astype(np.float32)
stepio = popart.PyStepIO({ip: ip_data}, anchors)
session.run(stepio)
assert (np.array_equal(anchors[d1], anchors[d2]) is not True)
def return_options(anchorDict):
opts = popart.SessionOptions()
if anchorDict["Pipelining"]:
opts.enablePipelining = True
if anchorDict["AccumulationFactor"] > 1:
opts.enableGradientAccumulation = True
opts.accumulationFactor = anchorDict["AccumulationFactor"]
ipus = 1
if anchorDict["Pipelining"] == False:
ipus = 1 * anchorDict["ReplicationFactor"]
else:
opts.virtualGraphMode = popart.VirtualGraphMode.Auto
ipus = 2 * anchorDict["ReplicationFactor"]
if anchorDict["ReplicationFactor"] > 1:
opts.replicatedGraphCount = anchorDict["ReplicationFactor"]
opts.enableReplicatedGraphs = True
if tu.ipu_available(ipus):
device = tu.create_test_device(numIpus=ipus)
else:
print("No IPUS available for test options.")
return None, None
else:
device = tu.create_test_device(numIpus=ipus)
print("device: ", device)
return opts, device
def test_enum_specfic_devices():
"""Test that enumerating specific, per type and count works.
"""
ipu_counts = [1, 2, 4, 8, 16]
ipu_types = [popart.DeviceType.IpuModel, popart.DeviceType.Cpu]
if tu.ipu_available():
ipu_types += [popart.DeviceType.Ipu]
deviceManager = popart.DeviceManager()
for count, type_ in itertools.product(ipu_counts, ipu_types):
devices = deviceManager.enumerateDevices(
pattern=popart.SyncPattern.Full, numIpus=count, deviceType=type_)
for device in devices:
assert device.numIpus == count
assert isinstance(device.type, type(type_))
def test_pipelined_recomputed_dropout():
dsize = 10
ratio = 0.5
ipus = 4
layers = 4
batches_per_step = 7
# Ensure inputs in range [1.0, 2.0] to ensure comparing with 0 is valid
ip_shape = [dsize]
ip_data = np.full([batches_per_step] + ip_shape, 1).astype(np.float32)
dropouts = []
dropoutGrads = []
dropoutInputs = []
dropoutOutputs = []
builder = popart.Builder()
ip = builder.addInputTensor(popart.TensorInfo("FLOAT", ip_shape))
def add_layer(layer_input, vgraph_num):
# This is to get the output of the dropout in the bwd pass.
# D_next_layer_in also includes the gradient of the AddOp.
identity0 = builder.aiOnnx.identity([layer_input])
builder.virtualGraph(identity0, vgraph_num)
[dropout0] = builder.aiOnnx.dropout([identity0],
num_outputs=1,
ratio=ratio)
builder.virtualGraph(dropout0, vgraph_num)
# the input to the forward pass dropout
dropoutInputs.append(identity0)
# the input to the backward pass dropout
dropoutInputs.append(popart.reservedGradientPrefix() + dropout0)
# the output of the backward pass dropout
dropoutGrads.append(popart.reservedGradientPrefix() + identity0)
# the output of the forward pass dropout
dropouts.append(dropout0)
relu0 = builder.aiOnnx.relu([dropout0])
builder.virtualGraph(relu0, vgraph_num)
# This ensures the all input elements to the dropouts, in both
# the forward and backward passes, will be non-zero.
add0 = builder.aiOnnx.add([layer_input, dropout0])
builder.virtualGraph(add0, vgraph_num)
return add0
# construct a graph of `layers` number of layers
# with each layer on a different IPU.
next_layer_in = ip
for vgraph in range(layers):
next_layer_in = add_layer(next_layer_in, vgraph)
out = next_layer_in
# TODO: use the tu.requires_ipu decorator
if tu.ipu_available(ipus):
device = tu.create_test_device(numIpus=ipus)
else:
pytest.skip("Test needs to run on IPU, but none are available")
dfAnchors = {}
for x in dropouts + dropoutGrads + dropoutInputs:
dfAnchors[x] = popart.AnchorReturnType("All")
dataFlow = popart.DataFlow(batches_per_step, dfAnchors)
loss = builder.aiGraphcore.identityloss([out])
builder.virtualGraph(loss, layers - 1)
userOptions = popart.SessionOptions()
userOptions.virtualGraphMode = popart.VirtualGraphMode.Manual
userOptions.enablePipelining = True
userOptions.autoRecomputation = popart.RecomputationType.Pipeline
session = popart.TrainingSession(fnModel=builder.getModelProto(),
dataFlow=dataFlow,
optimizer=popart.ConstSGD(0.1),
loss=loss,
userOptions=userOptions,
deviceInfo=device)
session.prepareDevice()
session.weightsFromHost()
anchors = session.initAnchorArrays()
session.setRandomSeed(0)
stepio = popart.PyStepIO({ip: ip_data}, anchors)
session.run(stepio)
print(anchors.keys())
# Check that none of the elements of the dropout inputs are zero
for tid in dropoutInputs:
x = anchors[tid]
print(f'{tid}: {x}')
zero = np.zeros(x.shape)
assert not np.any(np.equal(x, zero)), \
f'Some elements of dropout input {tid} are zero'
print()
# For each dropout, check that the masked out elements are the same
# in the forward and backward passes.
for fwdId, bwdId in zip(dropouts, dropoutGrads):
print(f'{fwdId}:\n{np.sign(anchors[fwdId])}')
print(f'{bwdId}:\n{np.sign(anchors[bwdId])}')
lhs = np.sign(anchors[fwdId])
rhs = np.sign(anchors[bwdId])
assert np.array_equal(lhs, rhs), \
f'{fwdId} and {bwdId} did not use the same dropout mask'
print()
