def prepare_model(cfg):
opt = argparse.ArgumentParser()
opt.weights = 'weights/yolov4-p5-sd.pt'
model = Yolov4P5(cfg)
model.eval()
model = load_and_fuse_pretrained_weights(model, opt)
model.optimize_for_inference()
ipu_opts = ipu_options(opt, cfg, model)
return inferenceModel(model, ipu_opts)
def test_input_plain_list():
model = torch.nn.ReLU()
inference_model = poptorch.inferenceModel(model)
with pytest.raises(TypeError) as excinfo:
inference_model([torch.tensor([1])])
assert (str(
excinfo.value) == "Lists are not supported as input arguments, "
"including when nested in tuples.\n"
"Received list input = [tensor([1])]")
def test_explicit_weightcopy_inference(self):
x = torch.ones([1] + list(datasets_info["cifar10"]["in"]))
opts = TestWeightSync.create_opts()
model = models.get_model(opts, datasets_info["imagenet"], pretrained=True)
model.eval()
poptorch_model = poptorch.inferenceModel(model)
y = model(x)
for _ in range(10):
poptorch_model.copyWeightsToDevice()
y_poptorch = poptorch_model(x)
assert torch.allclose(y, y_poptorch, atol=0.0001)
def test_unused_tuple():
class SimpleAdder(nn.Module):
def forward(self, x, y, z): # pylint: disable=unused-argument
return x + y
model = SimpleAdder()
inference_model = poptorch.inferenceModel(model)
t1 = torch.tensor([1.])
t2 = torch.tensor([2.])
z = (torch.tensor([1.]), torch.tensor([1.]))
inference_model(t1, t2, z)
def test_many_implicit_cast_one_less_than(input_type):
class OneLessThan(nn.Module):
def forward(self, x):
return 1 < x # pylint: disable=misplaced-comparison-constant
model = poptorch.inferenceModel(OneLessThan())
t = torch.tensor([2.5, -1, 2.0, 550.4], dtype=input_type)
np.testing.assert_equal(
model(t).numpy(), np.array([True, False, True, True]))
def test_many_implicit_cast_less_than_one(input_type):
class LessThanOne(nn.Module):
def forward(self, x):
return x < 1
model = poptorch.inferenceModel(LessThanOne())
t = torch.tensor([2.5, -1, 2.0, 550.4], dtype=input_type)
np.testing.assert_equal(
model(t).numpy(), np.array([False, True, False, False]))
def test_ExecutableCaching(capfd):
poptorch.setLogLevel(1) # Force debug logging
class Model(torch.nn.Module):
def forward(self, x):
return x * 6
with tempfile.TemporaryDirectory() as cache:
opts = poptorch.Options()
opts.enableExecutableCaching(cache)
m = poptorch.inferenceModel(Model(), opts)
m.compile(torch.rand(2, 3))
m.destroy()
log = helpers.LogChecker(capfd)
log.assert_contains("set enableEngineCaching to value true")
assert os.listdir(), "No executable saved in the cache"
n = poptorch.inferenceModel(Model(), opts)
n.compile(torch.rand(2, 3))
log = helpers.LogChecker(capfd)
log.assert_contains("set enableEngineCaching to value true")
def test_invalid_multiconv_empty():
class Model(torch.nn.Module):
def forward(self, x):
with poptorch.MultiConv():
return torch.pow(x, 2)
m = Model()
poptorch_model = poptorch.inferenceModel(m)
msg = "Unexpected end_multi_conv"
with pytest.raises(RuntimeError, match=msg):
poptorch_model(torch.ones(2, 2))
def test_octconv_block(use_multi):
block = OctConvBlock(3, 6, (0., .5), use_multi=use_multi)
# N, C, H, W
x = torch.randn(5, 3, 10, 10)
out = block(x)
pop_block = poptorch.inferenceModel(block)
pop_out = pop_block(x)
for (cpu, ipu) in zip(out, pop_out):
torch.testing.assert_allclose(cpu, ipu)
def test_access_parameters(use_half):
torch.manual_seed(42)
# 10 Batches of 10.
input = torch.randn(10, 10)
# 10 batches of 1
label = torch.randint(0, 10, [1])
label = label.expand([10])
class Model(torch.nn.Module):
def __init__(self):
super().__init__()
self.linear = torch.nn.Linear(10, 10)
def forward(self, x):
return self.linear(x)
model = Model()
if use_half:
model.half()
input = input.half()
# Run on IPU batch size 1 * 10 popart batches.
opts = poptorch.Options().deviceIterations(10)
poptorch_model = helpers.trainingModelWithLoss(
model, options=opts, loss=torch.nn.CrossEntropyLoss())
original_weights = str(model.linear.weight)
inference = poptorch.inferenceModel(model)
# Run all 10 batches as batchsize 10.
out = inference(input)
assert original_weights == str(model.linear.weight)
# Sanity check we weren't already matching the label.
assert not torch.equal(torch.argmax(out.int(), dim=1), label)
for _ in range(0, 1000):
_, loss = poptorch_model(input, label)
# Each batch should NOT report its own loss. As by default training model should have a "Final" anchor.
assert len(loss.size()) == 0
assert original_weights != str(poptorch_model.model.linear.weight)
# Run with trained weights.
out = inference(input)
# Check we are now equal with labels.
assert torch.equal(torch.argmax(out.int(), dim=1), label)
def test_bert_medium_result():
torch.manual_seed(42)
# Bert small.
pretrained_weights = 'mrm8488/bert-medium-finetuned-squadv2'
model = transformers.BertForQuestionAnswering.from_pretrained(
pretrained_weights)
tokenizer = transformers.BertTokenizer.from_pretrained(
pretrained_weights, return_token_type_ids=True)
context = """Edinburgh is Scotland's compact, hilly capital."""
question = "What is the capital of Scotland?"
encoding = tokenizer.encode_plus(question, context)
mask = encoding["attention_mask"]
ins = encoding["input_ids"]
input_ids = torch.tensor([ins, ins])
attention_mask = torch.tensor([mask, mask])
start_scores_native, end_scores_native = model(
input_ids, attention_mask=attention_mask)
opts = poptorch.Options()
opts.deviceIterations(2)
model.bert.embeddings.position_embeddings = poptorch.BeginBlock(
model.bert.embeddings.position_embeddings, ipu_id=1)
inference_model = poptorch.inferenceModel(model, opts)
start_score_pop, end_scores_pop = inference_model(input_ids,
attention_mask)
# Longer sequences begin to accumulate more floating point error.
assert torch.allclose(start_scores_native,
start_score_pop,
rtol=1e-02,
atol=1e-02)
assert torch.allclose(end_scores_native,
end_scores_pop,
rtol=1e-02,
atol=1e-02)
assert torch.argmax(start_score_pop), torch.argmax(start_scores_native)
assert torch.argmax(end_scores_pop), torch.argmax(end_scores_native)
# Convert to string (Only check the first result as we've already established the two were identical)
ans_tokens = ins[torch.argmax(start_score_pop[0]
):torch.argmax(end_scores_pop[0]) + 1]
answer_tokens = tokenizer.convert_ids_to_tokens(ans_tokens)
answer_tokens_to_string = tokenizer.convert_tokens_to_string(answer_tokens)
assert answer_tokens_to_string == 'edinburgh'
def test_many_implicit_cast_greater_than(input_1_type, input_2_type):
class GreaterThan(nn.Module):
def forward(self, x, y):
return x > y
model = poptorch.inferenceModel(GreaterThan())
t1 = torch.tensor([1, -1, 2.0, 550.4], dtype=input_1_type)
t2 = torch.tensor([2.4, 2, 1.0, 32.4], dtype=input_2_type)
np.testing.assert_equal(
model(t1, t2).numpy(), np.array([False, False, True, True]))
def test_linear():
model = torch.nn.Linear(20, 30)
x = torch.randn(128, 20)
# Run on CPU.
nativeOut = model(x)
# Run on IPU.
poptorch_model = poptorch.inferenceModel(model)
poptorch_out = poptorch_model(x)
assert nativeOut.size() == poptorch_out.size()
torch.testing.assert_allclose(nativeOut, poptorch_out)
def test_embedding():
model = torch.nn.Embedding(10, 3)
x = torch.LongTensor([[1, 2, 4, 5], [4, 3, 2, 9]])
# Run on CPU.
nativeOut = model(x)
# Run on IPU.
poptorch_model = poptorch.inferenceModel(model)
poptorch_out = poptorch_model(x)
assert nativeOut.size() == poptorch_out.size()
assert torch.equal(nativeOut, poptorch_out)
def test_PoissonNLLLoss_direct(reduction, log_input, full):
torch.manual_seed(42)
model = torch.nn.PoissonNLLLoss(log_input, full, reduction=reduction)
poptorch_model = poptorch.inferenceModel(model)
target = torch.poisson(torch.rand(10) * 5)
input = torch.empty(10).uniform_()
native_out = model(input, target)
poptorch_out = poptorch_model(input, target)
torch.testing.assert_allclose(native_out, poptorch_out)
def test_ipu_id_selection():
class Network(nn.Module):
def forward(self, x, y):
return x + y
model = Network()
# Force-disable the IPU model
opts = poptorch.Options().useIpuId(0)
inference_model = poptorch.inferenceModel(model, opts)
x = torch.ones(2)
y = torch.zeros(2)
inference_model(x, y)
def test_SmoothL1Loss_direct(reduction):
torch.manual_seed(42)
model = torch.nn.SmoothL1Loss(reduction=reduction)
poptorch_model = poptorch.inferenceModel(model)
input = torch.randn(10)
target = torch.empty(10).uniform_()
native_out = model(input, target)
poptorch_out = poptorch_model(input, target)
torch.testing.assert_allclose(native_out, poptorch_out)
def test_lstm_batch_first():
torch.manual_seed(42)
numHidden = 5
inputSize = 3
lstm = nn.LSTM(3, numHidden, batch_first=True)
ipuLstm = poptorch.inferenceModel(lstm)
inputs = [torch.randn(1, inputSize) for _ in range(5)]
# Add the extra 2nd dimension
inputs = torch.cat(inputs).view(1, len(inputs), -1)
hidden = (torch.randn(1, 1, numHidden), torch.randn(1, 1, numHidden))
out = lstm(inputs, hidden)
ipuOut = ipuLstm(inputs, hidden)
assert poptorch.testing.allclose(out, ipuOut)
def test_input_nested_list():
model = torch.nn.ReLU()
inference_model = poptorch.inferenceModel(model)
with pytest.raises(TypeError) as excinfo:
inference_model((torch.tensor([1]), torch.tensor([2]),
(torch.tensor([3]), [torch.tensor([4])],
torch.tensor([5])), torch.tensor([6])))
assert (str(
excinfo.value) == "Lists are not supported as input arguments, "
"including when nested in tuples.\n"
"Received list input[2][1] = [tensor([4])]")
def test_input_nested_dict():
model = torch.nn.ReLU()
inference_model = poptorch.inferenceModel(model)
with pytest.raises(TypeError) as excinfo:
inference_model(
(torch.tensor([1]), torch.tensor([2]), (torch.tensor([3]), {
'b': torch.tensor([4])
}, torch.tensor([5])), torch.tensor([6])))
assert (str(excinfo.value) == "Dictionaries are not supported as input "
"arguments, including when nested in tuples."
"\nReceived dict input[2][1] = "
"{'b': tensor([4])}")
def test_BCE_direct(reduction):
torch.manual_seed(42)
model = torch.nn.BCELoss(reduction=reduction)
poptorch_model = poptorch.inferenceModel(model)
for _ in range(0, 10):
target = torch.empty(10).random_(2)
input = torch.empty(10).uniform_()
groundTruth = model(input, target)
poptorch_out = poptorch_model(input, target)
assert torch.allclose(groundTruth, poptorch_out)
def prepare_model(cfg, debugging_nms=False):
opt = argparse.ArgumentParser()
opt.weights = os.environ['PYTORCH_APPS_DETECTION_PATH'] + '/weights/yolov4-p5-sd.pt'
model = Yolov4P5(cfg, debugging_nms=debugging_nms)
model.eval()
model = load_and_fuse_pretrained_weights(model, opt)
model.optimize_for_inference()
if cfg.model.ipu:
ipu_opts = ipu_options(opt, cfg, model)
return inferenceModel(model, ipu_opts)
else:
return model
def test_KLDiv_direct(reduction, log_target):
torch.manual_seed(42)
model = torch.nn.KLDivLoss(reduction=reduction, log_target=log_target)
poptorch_model = poptorch.inferenceModel(model)
# 2D Tensors to test batchmean
target = torch.empty(3, 10).uniform_()
input = torch.randn(3, 10)
native_out = model(input, target)
poptorch_out = poptorch_model(input, target)
torch.testing.assert_allclose(native_out, poptorch_out)
def test_layerNormScalar():
torch.manual_seed(42)
input = torch.randn([3, 2, 5, 2])
layerNorm = nn.LayerNorm(2)
# Run pytorch native on CPU.
nativeOutput = layerNorm(input)
# Run on IPU.
ipuModel = poptorch.inferenceModel(layerNorm)
poptorchOut = ipuModel(input)
assert torch.allclose(poptorchOut, nativeOutput)
def test_upsample(scale_factor, input_shape):
mode = "nearest" # Other modes not supported by Popart
model = torch.nn.Upsample(scale_factor=scale_factor, mode=mode)
x = torch.randn(*input_shape)
# Run on CPU.
nativeOut = model(x)
# Run on IPU.
poptorch_model = poptorch.inferenceModel(model)
poptorch_out = poptorch_model(x)
assert nativeOut.size() == poptorch_out.size()
torch.testing.assert_allclose(nativeOut, poptorch_out)
def get_model_and_loader(opt: argparse.ArgumentParser,
cfg: yacs.config.CfgNode):
"""Prepares the model and gets a new loader for the model.
Parameters:
opt: opt object containing options introduced in the command line
cfg: yacs object containing the config
Returns:
model[Detector]: a torch Detector Model
loader[DataLoader]: a torch or poptorch DataLoader containing the specified dataset on "cfg"
"""
# Create model
model = Yolov4P5(cfg)
if cfg.model.mode == "train":
model.train()
else:
model.eval()
# Load weights and fuses some batch normalizations with some convolutions
if cfg.model.normalization == 'batch':
if opt.weights:
print("loading pretrained weights")
model = load_and_fuse_pretrained_weights(model, opt)
model.optimize_for_inference()
# Create the specific ipu options if cfg.model.ipu
ipu_opts = ipu_options(opt, cfg, model) if cfg.model.ipu else None
# Creates the loader
loader = get_loader(opt, cfg, ipu_opts)
# Calls the poptorch wrapper and compiles the model
if cfg.model.ipu:
if cfg.model.mode == "train":
model = trainingModel(model, ipu_opts)
else:
model = inferenceModel(model, ipu_opts)
try:
img, _, _, _ = next(iter(loader))
model.compile(img)
warm_up_iterations = 100
for _ in range(warm_up_iterations):
_ = model(img)
except Exception as e:
print(e.args)
exit(0)
return model, loader
def test_explicit_deletion(use_half):
class ExampleModel(torch.nn.Module):
def __init__(self):
super().__init__()
self.bias = torch.nn.Parameter(torch.zeros(()))
def forward(self, x):
x += 1
# It is important to make sure the result of the print is used.
x = poptorch.ipu_print_tensor(x)
return x + self.bias
def custom_loss(output, target):
# Mean squared error with a scale
loss = output - target
loss = loss * loss * 5
return poptorch.identity_loss(loss, reduction="mean")
class ExampleModelWithCustomLoss(torch.nn.Module):
def __init__(self):
super().__init__()
self.model = ExampleModel()
def forward(self, input, target=None):
out = self.model(input)
if target is not None:
return out, custom_loss(out, target)
return out
opts = poptorch.Options()
# Both models will use the same IPU device.
opts.useIpuId(1)
model = ExampleModelWithCustomLoss()
input = torch.tensor([1.0, 2.0, 3.0])
target = torch.tensor([30.0, 40.0, 50.0])
if use_half:
model.half()
input = input.half()
target = target.half()
training_model = poptorch.trainingModel(model, opts)
inference_model = poptorch.inferenceModel(model, opts)
training_model(input=input, target=target)
training_model.destroy()
inference_model(input)
def execute_and_check_wrapper(model, input, check_shape_only=False):
# Run on CPU.
nativeOut = model(input)
# Run on IPU.
poptorch_model = poptorch.inferenceModel(model)
poptorch_out = poptorch_model(input)
print(nativeOut.size())
if not check_shape_only:
torch.testing.assert_allclose(poptorch_out, nativeOut)
else:
# This is due to adaptive pooling's process essentially being an implementation detail.
assert poptorch_out.size() == nativeOut.size()
def test_layerNorm():
torch.manual_seed(42)
for i in range(1, 4):
input = torch.randn([3, 2, 5, 2])
layerNorm = nn.LayerNorm(input.size()[i:])
# Run pytorch native on CPU.
nativeOutput = layerNorm(input)
# Run on IPU.
ipuModel = poptorch.inferenceModel(layerNorm)
poptorchOut = ipuModel(input)
assert torch.allclose(poptorchOut, nativeOutput)
def test_inference(self):
self.cfg.inference.nms = False
model = Yolov4P5(self.cfg)
model = inferenceModel(model.half().eval())
y = model(self.input_tensor)
expected_output_size = model.output_shape((64, 64))
p3 = expected_output_size['p3']
p4 = expected_output_size['p4']
p5 = expected_output_size['p5']
assert y[0].shape == torch.Size([p3[0], p3[1] * p3[2] * p3[3], p3[4]])
assert y[1].shape == torch.Size([p4[0], p4[1] * p4[2] * p4[3], p4[4]])
assert y[2].shape == torch.Size([p5[0], p5[1] * p5[2] * p5[3], p5[4]])