def test_dropout_training7(): dsize = 100 ratio = 0.2 builder = popart.Builder() ip = builder.addInputTensor(popart.TensorInfo("FLOAT", [dsize])) d__ip = popart.reservedGradientPrefix() + ip [d1] = builder.aiOnnx.dropout([ip], num_outputs=1, ratio=ratio) [d2] = builder.aiOnnx.dropout([ip], num_outputs=1, ratio=ratio) out = builder.aiOnnx.add([d1, d2]) out = builder.aiGraphcore.identityloss([out]) builder.addOutputTensor(out) if tu.ipu_available(): device = tu.create_test_device() else: pytest.skip("Test needs to run on IPU, but none are available") session, anchors = get_session(anchorIds=[d1, d2], proto=builder.getModelProto(), device=device, loss=out) # Same data for each batch ip_data = np.random.random_sample(dsize).astype(np.float32) stepio = popart.PyStepIO({ip: ip_data}, anchors) session.run(stepio) assert (np.array_equal(anchors[d1], anchors[d2]) is not True)
def return_options(anchorDict): opts = popart.SessionOptions() if anchorDict["Pipelining"]: opts.enablePipelining = True if anchorDict["AccumulationFactor"] > 1: opts.enableGradientAccumulation = True opts.accumulationFactor = anchorDict["AccumulationFactor"] ipus = 1 if anchorDict["Pipelining"] == False: ipus = 1 * anchorDict["ReplicationFactor"] else: opts.virtualGraphMode = popart.VirtualGraphMode.Auto ipus = 2 * anchorDict["ReplicationFactor"] if anchorDict["ReplicationFactor"] > 1: opts.replicatedGraphCount = anchorDict["ReplicationFactor"] opts.enableReplicatedGraphs = True if tu.ipu_available(ipus): device = tu.create_test_device(numIpus=ipus) else: print("No IPUS available for test options.") return None, None else: device = tu.create_test_device(numIpus=ipus) print("device: ", device) return opts, device
def test_enum_specfic_devices(): """Test that enumerating specific, per type and count works. """ ipu_counts = [1, 2, 4, 8, 16] ipu_types = [popart.DeviceType.IpuModel, popart.DeviceType.Cpu] if tu.ipu_available(): ipu_types += [popart.DeviceType.Ipu] deviceManager = popart.DeviceManager() for count, type_ in itertools.product(ipu_counts, ipu_types): devices = deviceManager.enumerateDevices( pattern=popart.SyncPattern.Full, numIpus=count, deviceType=type_) for device in devices: assert device.numIpus == count assert isinstance(device.type, type(type_))
def test_pipelined_recomputed_dropout(): dsize = 10 ratio = 0.5 ipus = 4 layers = 4 batches_per_step = 7 # Ensure inputs in range [1.0, 2.0] to ensure comparing with 0 is valid ip_shape = [dsize] ip_data = np.full([batches_per_step] + ip_shape, 1).astype(np.float32) dropouts = [] dropoutGrads = [] dropoutInputs = [] dropoutOutputs = [] builder = popart.Builder() ip = builder.addInputTensor(popart.TensorInfo("FLOAT", ip_shape)) def add_layer(layer_input, vgraph_num): # This is to get the output of the dropout in the bwd pass. # D_next_layer_in also includes the gradient of the AddOp. identity0 = builder.aiOnnx.identity([layer_input]) builder.virtualGraph(identity0, vgraph_num) [dropout0] = builder.aiOnnx.dropout([identity0], num_outputs=1, ratio=ratio) builder.virtualGraph(dropout0, vgraph_num) # the input to the forward pass dropout dropoutInputs.append(identity0) # the input to the backward pass dropout dropoutInputs.append(popart.reservedGradientPrefix() + dropout0) # the output of the backward pass dropout dropoutGrads.append(popart.reservedGradientPrefix() + identity0) # the output of the forward pass dropout dropouts.append(dropout0) relu0 = builder.aiOnnx.relu([dropout0]) builder.virtualGraph(relu0, vgraph_num) # This ensures the all input elements to the dropouts, in both # the forward and backward passes, will be non-zero. add0 = builder.aiOnnx.add([layer_input, dropout0]) builder.virtualGraph(add0, vgraph_num) return add0 # construct a graph of `layers` number of layers # with each layer on a different IPU. next_layer_in = ip for vgraph in range(layers): next_layer_in = add_layer(next_layer_in, vgraph) out = next_layer_in # TODO: use the tu.requires_ipu decorator if tu.ipu_available(ipus): device = tu.create_test_device(numIpus=ipus) else: pytest.skip("Test needs to run on IPU, but none are available") dfAnchors = {} for x in dropouts + dropoutGrads + dropoutInputs: dfAnchors[x] = popart.AnchorReturnType("All") dataFlow = popart.DataFlow(batches_per_step, dfAnchors) loss = builder.aiGraphcore.identityloss([out]) builder.virtualGraph(loss, layers - 1) userOptions = popart.SessionOptions() userOptions.virtualGraphMode = popart.VirtualGraphMode.Manual userOptions.enablePipelining = True userOptions.autoRecomputation = popart.RecomputationType.Pipeline session = popart.TrainingSession(fnModel=builder.getModelProto(), dataFlow=dataFlow, optimizer=popart.ConstSGD(0.1), loss=loss, userOptions=userOptions, deviceInfo=device) session.prepareDevice() session.weightsFromHost() anchors = session.initAnchorArrays() session.setRandomSeed(0) stepio = popart.PyStepIO({ip: ip_data}, anchors) session.run(stepio) print(anchors.keys()) # Check that none of the elements of the dropout inputs are zero for tid in dropoutInputs: x = anchors[tid] print(f'{tid}: {x}') zero = np.zeros(x.shape) assert not np.any(np.equal(x, zero)), \ f'Some elements of dropout input {tid} are zero' print() # For each dropout, check that the masked out elements are the same # in the forward and backward passes. for fwdId, bwdId in zip(dropouts, dropoutGrads): print(f'{fwdId}:\n{np.sign(anchors[fwdId])}') print(f'{bwdId}:\n{np.sign(anchors[bwdId])}') lhs = np.sign(anchors[fwdId]) rhs = np.sign(anchors[bwdId]) assert np.array_equal(lhs, rhs), \ f'{fwdId} and {bwdId} did not use the same dropout mask' print()