def test_load_onnx_model_from_other_builder(tmpdir):
# Run the first builder
builder = popart.Builder()
shape = popart.TensorInfo("FLOAT", [2])
i1 = builder.addInputTensor(shape)
i2 = builder.addInputTensor(shape)
o = builder.aiOnnx.add([i1, i2])
builder.addOutputTensor(o)
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
session = popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
deviceInfo=getDevice())
anchors = session.initAnchorArrays()
session.prepareDevice()
inputs = {
i1: np.array([1, 2], dtype=np.float32),
i2: np.array([3, 4], dtype=np.float32)
}
stepio = popart.PyStepIO(inputs, anchors)
session.run(stepio)
assert (np.array_equal(anchors[o], [4, 6]))
# Run a builder that imports the model of the other builder and check the
# output is still the same
builder2 = popart.Builder(proto)
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
proto2 = builder.getModelProto()
session = popart.InferenceSession(fnModel=proto2,
dataFlow=dataFlow,
deviceInfo=getDevice())
anchors = session.initAnchorArrays()
session.prepareDevice()
inputs = {
i1: np.array([1, 2], dtype=np.float32),
i2: np.array([3, 4], dtype=np.float32)
}
stepio = popart.PyStepIO(inputs, anchors)
session.run(stepio)
assert (np.array_equal(anchors[o], [4, 6]))
def test_batchnorm_shapeinference(op_tester):
# create test data
d1 = np.random.rand(1, 3, 2, 2).astype(np.float32) * 100
scale = np.random.rand(3).astype(np.float32)
b = np.random.rand(3).astype(np.float32)
mean = np.random.rand(3).astype(np.float32)
var = np.random.rand(3).astype(np.float32)
epsilon = 1e-05
momentum = 0.1
builder = popart.Builder()
i1 = builder.addInputTensor(popart.TensorInfo(d1))
iScale = builder.addInitializedInputTensor(scale)
iB = builder.addInitializedInputTensor(b)
iMean = builder.addInitializedInputTensor(mean)
iVar = builder.addInitializedInputTensor(var)
o_y, o_mean, o_var, o_smean, o_svar = builder.aiOnnx.batchnormalization(
[i1, iScale, iB, iMean, iVar], 5, epsilon, momentum)
builder.addOutputTensor(o_y)
builder.addOutputTensor(o_mean)
builder.addOutputTensor(o_var)
builder.addOutputTensor(o_smean)
builder.addOutputTensor(o_svar)
lossId = builder.aiGraphcore.identityloss([o_y])
proto = builder.getModelProto()
anchors = [o_y, o_mean, o_var, o_smean, o_svar]
art = popart.AnchorReturnType("All")
dataFlow = popart.DataFlow(1, {a: art for a in anchors})
device = tu.create_test_device()
options = popart.SessionOptions()
options.enableStochasticRounding = False
# store the shapes here to make sure we are checking shapes
# before the IR is complete (i.e. testing onnx shape inference)
shapes = []
for a in anchors:
shapes.append(tuple(builder.getTensorShape(a)))
session = popart.TrainingSession(fnModel=proto,
loss=lossId,
dataFlow=dataFlow,
deviceInfo=device,
optimizer=popart.ConstSGD(0.01),
userOptions=options)
anchors = session.initAnchorArrays()
session.prepareDevice()
inputs = {i1: d1}
stepio = popart.PyStepIO(inputs, anchors)
session.weightsFromHost()
session.run(stepio)
stepio = popart.PyStepIO(inputs, anchors)
# This tests the shape inference has run
for a, b in zip([o_y, o_mean, o_var, o_smean, o_svar], shapes):
assert anchors[a].shape == b
def test_batchnorm_repeated():
# create test data
d1 = np.random.rand(1, 3, 2, 2).astype(np.float16) * 100
scale = np.random.rand(3).astype(np.float16)
b = np.random.rand(3).astype(np.float16)
mean = np.random.rand(3).astype(np.float16)
var = np.random.rand(3).astype(np.float16)
epsilon = 1e-05
momentum = 0.1
builder = popart.Builder()
i1 = builder.addInputTensor(popart.TensorInfo(d1))
iScale = builder.addInitializedInputTensor(scale)
iB = builder.addInitializedInputTensor(b)
iMean = builder.addInitializedInputTensor(mean)
iVar = builder.addInitializedInputTensor(var)
(o_y, ) = builder.aiOnnx.batchnormalization([i1, iScale, iB, iMean, iVar],
1, epsilon, momentum)
builder.addOutputTensor(o_y)
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {o_y: popart.AnchorReturnType("All")})
device = tu.create_test_device()
options = popart.SessionOptions()
options.enableStochasticRounding = False
session = popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
deviceInfo=device,
userOptions=options)
anchors = session.initAnchorArrays()
session.prepareDevice()
inputs = {i1: d1}
stepio = popart.PyStepIO(inputs, anchors)
session.run(stepio)
first_result = np.copy(anchors[o_y])
for i in range(0, 10):
stepio = popart.PyStepIO(inputs, anchors)
session.run(stepio)
assert np.allclose(first_result, np.copy(anchors[o_y])) == True
def test_lstm_export_with_constantofshape(tmpdir):
np.random.seed(42)
torch.manual_seed(43)
class RNNNet(torch.nn.Module):
def __init__(self):
super(RNNNet, self).__init__()
hidden_size = 8
input_size = 18
self.lstm = torch.nn.LSTM(input_size=input_size,
hidden_size=hidden_size,
batch_first=True)
def forward(self, x):
x, (h, c) = self.lstm(x)
return x
net = RNNNet()
np_data = np.random.rand(1, 100, 18).astype(np.float32)
torch_data = torch.from_numpy(np_data)
torchOutput = net(torch_data).detach().numpy()
export_name = str(tmpdir / "lstm_small_repro.onnx")
torch.onnx.export(net,
torch_data,
export_name,
verbose=True,
input_names=['data'],
output_names=['tag'])
# Verify this model contains a ConstantOfShape op.
model = onnx.load(export_name)
nodes = model.graph.node
nodes = [i for i in nodes if i.op_type == 'ConstantOfShape']
assert len(nodes) > 0
inputShapeInfo = popart.InputShapeInfo()
inputShapeInfo.add("data", popart.TensorInfo("FLOAT", [1, 100, 18]))
anchors = {"tag": popart.AnchorReturnType("All")}
dataFlow = popart.DataFlow(1, anchors)
device = tu.create_test_device()
session = popart.InferenceSession(export_name,
dataFlow,
device,
inputShapeInfo=inputShapeInfo)
session.prepareDevice()
inferenceAnchors = session.initAnchorArrays()
stepio = popart.PyStepIO({"data": np_data}, inferenceAnchors)
session.run(stepio)
popartOutput = inferenceAnchors['tag']
assert torchOutput.shape == popartOutput.shape
assert np.allclose(torchOutput, popartOutput, atol=1e-07)
def test_replicated_with_multiple_batches_per_step():
replication_factor = 4
dsize = 100
batches_per_step = 2
session, ip, out, d__ip, anchors = get_replicated_dropout_session(
dsize=dsize,
num_layers=1,
ratio=0.3,
replication_factor=replication_factor,
batches_per_step=batches_per_step)
ip_data = np.ones([batches_per_step, replication_factor, dsize],
dtype=np.float32)
stepio = popart.PyStepIO({ip: ip_data}, anchors)
session.run(stepio)
ref_out = np.copy(anchors[out])
# Another call should produce different results
session.run(stepio)
o = anchors[out]
micro_batches = []
for batch_index in range(batches_per_step):
for replication_index in range(replication_factor):
x = o[batch_index][replication_index]
micro_batches.append(x)
# Check that none of the micro batch results are the same
for ai, bi in itertools.combinations(
[i for i in range(len(micro_batches))], 2):
a = micro_batches[ai]
b = micro_batches[bi]
assert not np.allclose(a, b)
def test_load_externally_saved_tensors():
"""
Test that initializer data can be saved in a separate file, and read into
the PopART IR in an InferenceSession (by observing an expected inference
result)
"""
builder = popart.Builder()
d1 = np.array([1, -1, 6]).astype(np.float32)
d2 = np.array([-8, 7, 4]).astype(np.float32)
i1 = builder.addInitializedInputTensor(d1)
i2 = builder.addInitializedInputTensor(d2)
o = builder.aiOnnx.add([i1, i2])
tmpdir = tempfile.mkdtemp()
tmpfile_tensors = os.path.join(tmpdir, "tensors.onnx")
tmpfile_model = os.path.join(tmpdir, "model.onnx")
builder.saveInitializersExternally([i1, i2], tmpfile_tensors)
builder.saveModelProto(tmpfile_model)
# Create builder from onnx model
builder = popart.Builder(tmpfile_model)
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
session = popart.InferenceSession(
fnModel=builder.getModelProto(),
dataFlow=dataFlow,
deviceInfo=popart.DeviceManager().createCpuDevice())
anchors = session.initAnchorArrays()
session.prepareDevice()
stepio = popart.PyStepIO({}, anchors)
session.run(stepio)
assert (np.array_equal(anchors[o], d1 + d2))
def test_dropout_training3():
dsize = 10000 # large input size to make statistical assumptions accurate
ratio = 0.2
session, ip, out, d__ip, anchors = get_dropout_session(dsize=dsize,
ratio=ratio)
# Ensure inputs in range [1.0, 2.0] to ensure comparing with 0 is valid
ip_data = np.random.random_sample(dsize).astype(np.float32) + 1
stepio = popart.PyStepIO({ip: ip_data}, anchors)
session.run(stepio)
t1 = torch.tensor(ip_data)
dropout = torch.nn.Dropout(p=ratio)
torchOut = dropout(t1).numpy()
# 1.
for onnxEl, torchEl in zip(anchors[out], torchOut):
if onnxEl != 0 and torchEl != 0:
assert np.isclose(onnxEl, torchEl)
# 2.
onnxDropoutProportion = np.count_nonzero(anchors[out]) / dsize
torchDropoutProportion = np.count_nonzero(torchOut) / dsize
assert (np.isclose(onnxDropoutProportion,
torchDropoutProportion,
atol=0.05))
assert (np.isclose(onnxDropoutProportion, 1 - ratio, atol=0.05))
def test_dropout_training_replicated_repeatable():
replication_factor = 4
dsize = 10
session, ip, out, d__ip, anchors = get_replicated_dropout_session(
dsize=dsize,
num_layers=1,
ratio=0.3,
replication_factor=replication_factor)
ip_data = np.ones([replication_factor, dsize], dtype=np.float32)
stepio = popart.PyStepIO({ip: ip_data}, anchors)
session.setRandomSeed(7)
session.run(stepio)
ref_out = np.copy(anchors[out])
ref_d__ip = np.copy(anchors[d__ip])
# Another call should produce different results
session.run(stepio)
assert not np.array_equal(ref_out, anchors[out])
assert not np.array_equal(ref_d__ip, anchors[d__ip])
# Resetting the seed should give the same results as the first run
session.setRandomSeed(7)
session.run(stepio)
assert np.array_equal(ref_out, anchors[out])
assert np.array_equal(ref_d__ip, anchors[d__ip])
def test_dropout_training4():
dsize = 10
ratio = 0.2
builder = popart.Builder()
ip = builder.addInputTensor(popart.TensorInfo("FLOAT", [dsize, dsize]))
d__ip = popart.reservedGradientPrefix() + ip
[d1] = builder.aiOnnx.dropout([ip], num_outputs=1, ratio=ratio)
# Matmul to change the layout -- ensures we are testing the dependency
# of random mask on the layout of the 'reference' dropout tensor
w = builder.addInitializedInputTensor(np.ones([dsize, dsize], np.float32))
out = builder.aiOnnx.matmul([d1, w])
out = builder.aiGraphcore.identityloss([out])
builder.addOutputTensor(out)
device = tu.create_test_device()
session, anchors = get_session(anchorIds=[d1, d__ip],
proto=builder.getModelProto(),
device=device,
loss=out)
# Ensure inputs in range [1.0, 2.0] to ensure comparing with 0 is valid
ip_data = np.random.random_sample((dsize, dsize)).astype(np.float32) + 1
stepio = popart.PyStepIO({ip: ip_data}, anchors)
session.run(stepio)
for fwdEl, bwdEl in zip(np.ndarray.flatten(anchors[d1]),
np.ndarray.flatten(anchors[d__ip])):
if fwdEl == 0:
assert bwdEl == 0
if bwdEl != 0:
assert fwdEl != 0
def test_replicated_allreduce():
input_data = np.array(range(10), dtype=np.float32)
replicatedGraphCount = 2
builder = popart.Builder()
t = builder.addInitializedInputTensor(input_data, "input")
o = builder.aiGraphcore.replicatedallreduce([t])
builder.addOutputTensor(o)
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
opts = popart.SessionOptions()
opts.enableReplicatedGraphs = True
opts.replicatedGraphCount = replicatedGraphCount
numIpus = 2
device = tu.create_test_device(numIpus=numIpus)
session = popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
deviceInfo=device)
session.prepareDevice()
anchors = session.initAnchorArrays()
inputs = {}
stepio = popart.PyStepIO(inputs, anchors)
session.run(stepio)
ground_truth = 2.0 * np.array(range(10), dtype=np.float32)
for i in range(replicatedGraphCount):
assert np.allclose(anchors[o][i], ground_truth)
def run(self, inputs):
#print(self.session)
# Create buffers to receive results from the execution
anchors = self.session.initAnchorArrays()
inputmap = {}
i = 0
for inp in self.model.graph.input:
isInitializer = False
for init in self.model.graph.initializer:
if inp.name == init.name:
isInitializer = True
break
if isInitializer == False:
inputmap[str(inp.name)] = inputs[i]
i = i + 1
stepio = popart.PyStepIO(inputmap, anchors)
self.session.run(stepio)
outputs = [i.name for i in self.model.graph.output]
outputs = [anchors[i] for i in outputs]
return outputs
def bert_process_infer_data(args, session, data, anchors, logits,
iteration: Iteration, start_times, end_times,
stepio):
if stepio is None:
stepio = popart.PyStepIO(data, anchors)
start = time.perf_counter()
session.run(stepio)
duration = time.perf_counter() - start
hw_cycles = session.getCycleCount() if args.report_hw_cycle_count else None
if args.gc_profile:
import gcprofile
gcprofile.save_popart_report(session)
sys.exit(0)
iteration.durations.append(duration)
mean_latency, min_latency, max_latency = compute_latency(
args, start_times, end_times)
if (iteration.count % iteration.steps_per_log) == 0:
status_string = \
f"Iteration: {iteration.count:6} " \
f"Duration: {np.average(iteration.durations):6.4f} s " \
f"Throughput: {np.average(iteration.throughput):6.1f} samples/s"
if mean_latency is not None:
status_string += f" Per-sample Latency: {mean_latency} {min_latency} {max_latency} seconds (mean min max)"
if hw_cycles is not None:
status_string += f" Cycles: {hw_cycles}"
logger.info(status_string)
iteration.count += 1
return [anchors[logit] for logit in logits]
def getAnchors(extraReduction):
builder = popart.Builder()
ip = builder.addInitializedInputTensor(ip_data)
lb = builder.addInputTensor("INT32", lshape)
sm = builder.aiOnnx.softmax([ip], axis=np.size(lshape))
if extraReduction == True:
nll = builder.aiGraphcore.nllloss(
[sm, lb], reduction=popart.ReductionType.NoReduction)
loss = builder.aiOnnx.reducesum([nll])
else:
loss = builder.aiGraphcore.nllloss(
[sm, lb], reduction=popart.ReductionType.Sum)
anchors = [popart.reservedGradientPrefix() + ip]
# Always test 'loss' too, except for when we want to test with
# the SoftmaxGradDirect pattern, which requires 'loss' to be
# anchored
if 'SoftmaxGradDirect' not in patternsList or 'NlllWithSoftmaxGradDirect' in patternsList:
anchors.append(loss)
session = popart.TrainingSession(
fnModel=builder.getModelProto(),
loss=loss,
dataFlow=popart.DataFlow(1, anchors),
optimizer=popart.ConstSGD(0.1),
deviceInfo=tu.create_test_device(),
patterns=popart.Patterns(patternsList))
session.prepareDevice()
session.weightsFromHost()
anchors = session.initAnchorArrays()
stepio = popart.PyStepIO({lb: lb_data.astype(np.int32)},
anchors)
session.run(stepio)
return anchors
def getAnchors(extraReduction):
builder = popart.Builder()
ip = builder.addInitializedInputTensor(ip_data)
if extraReduction == True:
l1 = builder.aiGraphcore.l1loss(
[ip], 0.1, reduction=popart.ReductionType.NoReduction)
loss = builder.aiOnnx.reducesum([l1])
else:
loss = builder.aiGraphcore.l1loss(
[ip], 0.1, reduction=popart.ReductionType.Sum)
anchors = [loss, popart.reservedGradientPrefix() + ip]
session = popart.TrainingSession(
fnModel=builder.getModelProto(),
loss=loss,
dataFlow=popart.DataFlow(1, anchors),
optimizer=popart.ConstSGD(0.1),
deviceInfo=tu.create_test_device(),
patterns=popart.Patterns(
popart.PatternsLevel.NoPatterns).enableRuntimeAsserts(
False))
session.prepareDevice()
session.weightsFromHost()
anchors = session.initAnchorArrays()
stepio = popart.PyStepIO({}, anchors)
session.run(stepio)
return anchors
def run_test():
proto, data, x, loss = model()
options = popart.SessionOptions()
dataFlow = popart.DataFlow(1, {x: popart.AnchorReturnType("ALL")})
with tu.create_test_device() as device:
session = popart.TrainingSession(fnModel=proto,
dataFlow=dataFlow,
deviceInfo=device,
userOptions=options,
loss=loss,
optimizer=popart.ConstSGD(0.01))
session.prepareDevice()
session.weightsFromHost()
anchors = session.initAnchorArrays()
stepio = popart.PyStepIO(data, anchors)
session.run(stepio)
ir = json.loads(
session._serializeIr(popart.IrSerializationFormat.JSON))
dropouts = [
op for op in ir['maingraph'] if op['type'] == 'Dropout'
]
assert (len(dropouts) <= 2)
device.detach()
return anchors[x]
def step(session, anchors, data, update_optimizer_lr=None):
if update_optimizer_lr is not None:
optimizer = popart.SGD(update_optimizer_lr)
session.updateOptimizer(optimizer)
stepio = popart.PyStepIO(data, anchors)
session.run(stepio)
def test_abort_conditional():
input_data = np.array([0], dtype=np.float32)
builder = popart.Builder()
t = builder.addInitializedInputTensor(input_data, "input")
o = builder.aiOnnx.abs([t])
builder.aiGraphcore.abort([o])
builder.addOutputTensor(o)
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
opts = popart.SessionOptions()
device = tu.create_test_device()
session = popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
deviceInfo=device)
session.prepareDevice()
anchors = session.initAnchorArrays()
inputs = {}
stepio = popart.PyStepIO(inputs, anchors)
session.run(stepio)
def run_model(session: popart.TrainingSession, anchors: Dict,
in_array: np.array,
label_array: np.array) -> Tuple[np.array, np.array, np.array]:
"""Run the model using the provided params
Args:
session (popart.TrainingSession): The compiled session to use
anchors (Dict): The anchors to test
in_array (np.array): The input array
label_array (np.array): The label array
Returns:
Tuple[np.array, np.array, np.array]: The results from the run.
"""
stepio = popart.PyStepIO(
{
"main_input_123": in_array,
"label_input_456": label_array
}, anchors)
session.weightsFromHost()
session.run(stepio)
session.weightsToHost()
return anchors["main_input_123"], anchors["Relu:0"], anchors[
"Nll:0"], anchors["MatMul:0"]
def test_abort_unconditional():
input_data = np.array(range(10), dtype=np.float32)
builder = popart.Builder()
t = builder.addInitializedInputTensor(input_data, "input")
o = builder.aiOnnx.abs([t])
builder.aiGraphcore.abort([])
builder.addOutputTensor(o)
proto = builder.getModelProto()
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
opts = popart.SessionOptions()
device = tu.create_test_device()
session = popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
deviceInfo=device)
session.prepareDevice()
anchors = session.initAnchorArrays()
inputs = {}
stepio = popart.PyStepIO(inputs, anchors)
with pytest.raises(popart.poplar_runtime_error) as e_info:
session.run(stepio)
assert (e_info.value.args[0].startswith("Abort Program"))
def getUpdatedWeights(decomposeGradSum):
opts = popart.SessionOptions()
opts.decomposeGradSum = decomposeGradSum
loss = builder.aiGraphcore.identityloss([actIn])
if doSharding == True:
builder.virtualGraph(loss, numLayers - 1)
opts.virtualGraphMode = popart.VirtualGraphMode.Manual
numIpus = numLayers
else:
numIpus = 1
device = tu.create_test_device(numIpus=numIpus)
session = popart.TrainingSession(fnModel=builder.getModelProto(),
dataFlow=popart.DataFlow(1, [w0]),
deviceInfo=device,
optimizer=popart.ConstSGD(0.1),
loss=loss,
userOptions=opts)
anchors = session.initAnchorArrays()
np.random.seed(1) # ensure same input vals between sessions
inputs = {in0: np.random.rand(*shape).astype('float32')}
stepio = popart.PyStepIO(inputs, anchors)
session.prepareDevice()
session.weightsFromHost()
session.run(stepio)
return anchors[w0]
def bert_process_data(args, session, labels, data, anchors, losses,
predictions, iteration: Iteration,
optimizer_factory: ScheduledOptimizerFactory):
labels_data = [data[label] for label in labels]
if not np.any([np.any(label) for label in labels_data]):
# Label may be all padding due to args.vocab_length being smaller than when the data was generated
return
stepio = popart.PyStepIO(data, anchors)
start = time.time()
session.run(stepio)
duration = time.time() - start
hw_cycles = session.getCycleCount() if args.report_hw_cycle_count else None
iteration.add_stats(duration, hw_cycles, labels_data, anchors, losses,
predictions)
if (iteration.count % iteration.steps_per_log) == 0:
iteration.report_stats()
utils.fetch_reports(args, session=session, execution=True)
# The following will only be true if:
# Learning rate mode is STEP and the current total step counter is in the schedule
# Learning rate mode is EPOCH and the current epoch has just changed to one in the schedule
if optimizer_factory.should_update(iteration):
optimizer = optimizer_factory.update_and_create(iteration)
session.updateOptimizerFromHost(optimizer)
iteration.count += 1
def bert_process_infer_data(args, session, data, anchors,
logits, iteration: Iteration):
start_times = defaultdict(list)
end_times = defaultdict(list)
if args.low_latency_inference and args.task == "SQUAD":
stepio = create_callback_stepio(data, anchors, start_times, end_times)
else:
stepio = popart.PyStepIO(data, anchors)
start = time.perf_counter()
session.run(stepio)
duration = time.perf_counter() - start
if args.gc_profile:
import gcprofile
gcprofile.save_popart_report(session)
sys.exit(0)
iteration.durations.append(duration)
mean_latency, min_latency, max_latency = compute_latency(args, start_times, end_times)
if (iteration.count % iteration.steps_per_log) == 0:
status_string = \
f"Iteration: {iteration.count:6} " \
f"Duration: {np.average(iteration.durations):6.4f} s " \
f"Throughput: {np.average(iteration.throughput):6.1f} samples/s"
if mean_latency is not None:
status_string += f" Per-sample Latency: {mean_latency} {min_latency} {max_latency} seconds (mean min max)"
logger.info(status_string)
iteration.count += 1
return [anchors[logit] for logit in logits]
def test_dropout_training7():
dsize = 100
ratio = 0.2
builder = popart.Builder()
ip = builder.addInputTensor(popart.TensorInfo("FLOAT", [dsize]))
d__ip = popart.reservedGradientPrefix() + ip
[d1] = builder.aiOnnx.dropout([ip], num_outputs=1, ratio=ratio)
[d2] = builder.aiOnnx.dropout([ip], num_outputs=1, ratio=ratio)
out = builder.aiOnnx.add([d1, d2])
out = builder.aiGraphcore.identityloss([out])
builder.addOutputTensor(out)
if tu.ipu_available():
device = tu.create_test_device()
else:
pytest.skip("Test needs to run on IPU, but none are available")
session, anchors = get_session(anchorIds=[d1, d2],
proto=builder.getModelProto(),
device=device,
loss=out)
# Same data for each batch
ip_data = np.random.random_sample(dsize).astype(np.float32)
stepio = popart.PyStepIO({ip: ip_data}, anchors)
session.run(stepio)
assert (np.array_equal(anchors[d1], anchors[d2]) is not True)
def run_ir(ir: pir.Ir, y: pir.Tensor):
ir_ = ir._pb_ir # Internal ir
y_d2h = pir.d2h_stream(y.shape, y.dtype, name="y_stream")
ops.host_store(y_d2h, y)
y_id = y_d2h.tensor_id()
dataFlow = popart.DataFlow(
batchesPerStep=1, anchorTensors={y_id: popart.AnchorReturnType("All")})
ir_.setDataFlow(dataFlow)
opts = ir_.getSessionOptions()
opts.useHostCopyOps = True
opts.enableExplicitMainLoops = True
opts.aliasZeroCopy = True
opts.explicitRecomputation = True
ir_.updateVertices()
session = popart.InferenceSession.fromIr(
ir=ir_, deviceInfo=tu.create_test_device())
session.prepareDevice()
# Create buffers for anchors
anchors = session.initAnchorArrays()
# Run the model
stepio = popart.PyStepIO(inputs={}, outputs=anchors)
session.weightsFromHost()
session.run(stepio)
y_ = anchors[y_id]
return y_
def test_dropout_training1():
dsize = 10
ratio = 0.2
d1 = np.random.rand(dsize).astype(np.float32)
builder = popart.Builder()
ip = builder.addInputTensor(popart.TensorInfo("FLOAT", [dsize]))
d__ip = popart.reservedGradientPrefix() + ip
[o1, o2] = builder.aiOnnx.dropout([ip], num_outputs=2, ratio=ratio)
out = builder.aiGraphcore.identityloss([o1])
builder.addOutputTensor(o1)
builder.addOutputTensor(o2)
session, anchors = get_session(anchorIds=[o1, o2, ip, d__ip],
proto=builder.getModelProto(),
device=tu.create_test_device(),
loss=out)
stepio = popart.PyStepIO({ip: d1}, anchors)
session.run(stepio)
# d1 * mask * (1/(1-ratio)) should give the same answer as popart implementation
reference = d1 * anchors[o2] * (1 / (1 - ratio))
assert (np.isclose(anchors[o1], reference)).all()
def run_test(proto, a, b, o, test):
device = tu.create_test_device()
# Describe how to run the model
dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")})
opts = popart.SessionOptions()
opts.enableEngineCaching = True
opts.cachePath = str(tmp_path / 'saved_graph')
# Create a session to compile and execute the graph
session = popart.InferenceSession(fnModel=proto,
dataFlow=dataFlow,
userOptions=opts,
deviceInfo=device)
# Compile graph
session.prepareDevice()
# Create buffers to receive results from the execution
anchors = session.initAnchorArrays()
# Generate some random input data
data_a = np.random.rand(1).astype(np.float32)
data_b = np.random.rand(1).astype(np.float32)
stepio = popart.PyStepIO({a: data_a, b: data_b}, anchors)
session.run(stepio)
assert test(data_a, data_b, anchors[o])
def test_randomuniform_repeatable(dtypes):
seed = 8
builder = popart.Builder()
out = builder.aiOnnx.randomuniform(shape=[10, 1], dtype=dtypes[1])
builder.addOutputTensor(out)
session = popart.InferenceSession(
fnModel=builder.getModelProto(),
dataFlow=popart.DataFlow(1, {out: popart.AnchorReturnType("All")}),
patterns=popart.Patterns(popart.PatternsLevel.All),
deviceInfo=tu.create_test_device())
session.prepareDevice()
session.setRandomSeed(seed)
session.weightsFromHost()
anchors = session.initAnchorArrays()
stepio = popart.PyStepIO({}, anchors)
session.run(stepio)
# need to copy the anchor as the next call to run will overwrite the data
run1_out = np.copy(anchors[out])
# Reset the seed to the same value and run the session again
session.setRandomSeed(seed)
session.run(stepio)
run2_out = np.copy(anchors[out])
assert np.array_equal(run1_out, run2_out)
def run_lstm_popart(onnx_file_name, inputs):
# generate a popart session
builder = popart.Builder(onnx_file_name)
outputs = builder.getOutputTensorIds()
dataFlow = popart.DataFlow(1, outputs)
device = tu.create_test_device(1)
s = popart.InferenceSession(fnModel=onnx_file_name,
dataFlow=dataFlow,
deviceInfo=device)
anchor_map = s.initAnchorArrays()
s.prepareDevice()
h0 = inputs[1]
c0 = inputs[2]
outs = []
for i in range(inputs[0].shape[0]):
input_data = inputs[0][i]
input_data = input_data.reshape(1, *input_data.shape)
input_map = {'X': input_data, 'initial_h': h0, 'initial_c': c0}
stepio = popart.PyStepIO(input_map, anchor_map)
s.run(stepio)
h0 = anchor_map['Y_h']
c0 = anchor_map['Y_c']
outs.append(np.copy(anchor_map['Y']))
outs = np.concatenate(outs)
return (outs, anchor_map['Y_h'], anchor_map['Y_c'])
def test_sgd_param_check():
"""
In this test we check that learning rate tensor, returned as an anchor,
matches the value supplied to the optimizer constructor
"""
lrName = popart.reservedDefaultScaledLearningRate0Prefix() + "FLOAT"
wdName = popart.reservedDefaultWeightDecayScaleFactor0Prefix() + "FLOAT"
lsName = popart.reservedLossScalingPrefix() + "FLOAT"
anchorNames = {
lrName: popart.AnchorReturnType("All"),
wdName: popart.AnchorReturnType("All"),
lsName: popart.AnchorReturnType("All")
}
# Just a placeholder optimizer. We overwrite the hyper-parameters in this
# test once the session is created
userSGD = popart.SGD({
"defaultLearningRate": (0.5, False),
"defaultWeightDecay": (0.6, False),
"lossScaling": (10.0, False)
})
stepSize = 2
session, inputsUserSgd = trainSession(anchorNames, userSGD, stepSize)
anchorsArrays = session.initAnchorArrays()
# train
numSteps = 3
learningRate = np.random.rand(numSteps).astype('float32')
weightDecay = np.random.rand(numSteps).astype('float32')
lossScaling = np.random.rand(numSteps).astype('float32')
for step in range(numSteps):
# Update learning rate parameter between training steps
stepLr = learningRate[step]
stepWd = weightDecay[step]
stepLs = lossScaling[step]
session.updateOptimizerFromHost(
popart.SGD({
"defaultLearningRate": (stepLr, False),
"defaultWeightDecay": (stepWd, False),
"lossScaling": (stepLs, False)
}))
stepio = popart.PyStepIO(inputsUserSgd, anchorsArrays)
session.run(stepio)
assert (np.array_equal(anchorsArrays[lsName][0], stepLs))
scaled = (stepLr / stepLs)
assert (np.array_equal(anchorsArrays[lrName][0], scaled))
# The weight decay tensor is scaled by lr on the host
# before training
scaled = 1 - (stepWd * stepLr)
assert (np.allclose(anchorsArrays[wdName][0], scaled))
def test_randomnormallike(dtypes):
seed = 8
data = np.zeros((1, 2, 10, 5), dtype=dtypes[0])
builder = popart.Builder()
T = builder.addInitializedInputTensor(data)
out = builder.aiOnnx.randomnormallike([T])
builder.addOutputTensor(out)
session = popart.InferenceSession(
fnModel=builder.getModelProto(),
dataFlow=popart.DataFlow(1, {out: popart.AnchorReturnType("All")}),
deviceInfo=tu.create_test_device())
session.prepareDevice()
session.setRandomSeed(seed)
session.weightsFromHost()
anchors = session.initAnchorArrays()
stepio = popart.PyStepIO({}, anchors)
session.run(stepio)
# Check that the output has the correct shape and dtype
assert anchors[out].shape == data.shape
assert anchors[out].dtype == data.dtype
# Check that the IR has RandomNormalLike replaced with RandomNormal
ir = json.loads(session._serializeIr(popart.IrSerializationFormat.JSON))
graph = ir['maingraph']
like = [op for op in graph if op['type'] == 'RandomNormalLike']
assert len(like) == 0, "Unexpected RandomNormalLike op in the IR."
rn = [op for op in graph if op['type'] == 'RandomNormal']
assert len(rn) == 1, "Expected one RandomNormal op in the IR."
