def test_reducemin_training(op_tester):
data = np.random.rand(2, 5, 3).astype(np.float32)
axes_list = [[0], [1], [2], [0, 1], [0, 2], [1, 2], [0, 1, 2],
USE_DEFAULT_AXES]
keepdims_list = [False, True]
def init_builder(builder):
result = []
axes_reduce = []
for axes, keepdims in itertools.product(axes_list, keepdims_list):
tensor = builder.addInputTensor(data)
if axes is USE_DEFAULT_AXES:
out = builder.aiOnnx.reducemin(
[tensor],
keepdims=keepdims,
debugContext="test_reducemin_default_{0}".format(keepdims))
else:
out = builder.aiOnnx.reducemin(
[tensor],
axes=axes,
keepdims=keepdims,
debugContext="test_reducemin_{0}_{1}".format(
axes, keepdims))
result.append(out)
result.append(popart.reservedGradientPrefix() + tensor)
axes_len = len(axes) if axes is not USE_DEFAULT_AXES else 3
axes_reduce.append(range(3 - (0 if keepdims else axes_len)))
sum = builder.aiOnnx.sum([
builder.aiOnnx.reducesum([r],
axes=axes,
keepdims=False,
debugContext="test_reducesum_all")
for r, axes in zip(result[0::2], axes_reduce)
],
debugContext="test_sum")
reshaped_sum = builder.aiOnnx.unsqueeze([sum],
axes=[0],
debugContext="test_reshape")
builder.addOutputTensor(reshaped_sum)
result = [
reshaped_sum,
popart.reservedGradientPrefix() + reshaped_sum
] + result
return result
def reference(ref_data):
result = []
for axes, keepdims in itertools.product(axes_list, keepdims_list):
tensor = torch.tensor(data, requires_grad=True)
out = tensor
sorted_axes = axes if axes is not USE_DEFAULT_AXES else [0, 1, 2]
sorted_axes.sort(reverse=True)
for dim in sorted_axes:
out = torch.min(out, dim=dim, keepdim=keepdims)[0]
result.append(out)
result.append(tensor)
sum = torch.unsqueeze(torch.sum(
torch.stack([torch.sum(r) for r in result[0::2]])),
dim=0)
d__o = ref_data.getOutputTensorGrad(0)
sum.backward(torch.tensor(d__o))
result[1::2] = [r.grad for r in result[1::2]]
result = [sum, sum.grad] + result
return result
op_tester.setPatterns(['OpToIdentity'], enableRuntimeAsserts=False)
op_tester.run(init_builder, reference, 'train')
def test_dynamicslice_overlap_correct(op_tester):
data = np.random.rand(10).astype(np.float32)
axes = [0]
sizes = [5]
def init_builder(builder):
tensor = builder.addInitializedInputTensor(data)
outputs = []
result = []
for sliceid in range(2):
index = builder.addInputTensor(np.asarray([sliceid * 3],
np.uint32))
out = builder.aiGraphcore.dynamicslice([tensor, index],
axes=axes,
sizes=sizes,
noOverlap=False)
out = builder.aiGraphcore.scale([out], float(1 + sliceid))
# TODO T46821: The shape given by getTensorShape is wrong
# Check the shape inference has run.
# assert builder.getTensorShape(out) == list(data.shape)
outputs.append(out)
result.append(out)
sum = builder.aiOnnx.sum(outputs)
sum = builder.aiOnnx.reducesum([sum], axes=[0], keepdims=False)
sum = builder.aiOnnx.unsqueeze([sum], axes=[0])
builder.addOutputTensor(sum)
result = [
sum,
popart.reservedGradientPrefix() + sum,
popart.reservedGradientPrefix() + tensor
] + result
return result
def reference(ref_data):
tensor = torch.tensor(data, requires_grad=True)
outputs = []
result = []
for sliceid in range(2):
out = tensor[sliceid * 3:sliceid * 3 + sizes[0]]
out = out * float(1 + sliceid)
outputs.append(out)
result.append(out)
sum = torch.unsqueeze(torch.sum(torch.stack(outputs)), dim=0)
d__o = ref_data.getOutputTensorGrad(0)
sum.backward(torch.tensor(d__o))
# Note: Comparison equal with noOverlap=False handles overlapping
# slices correctly (at higher computational cost). No correction needed.
tensor.grad += torch.tensor(
np.asarray([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]))
result = [sum, torch.tensor(d__o), tensor.grad] + result
return result
op_tester.setPatterns(popart.PatternsLevel.All, enableRuntimeAsserts=False)
op_tester.run(init_builder, reference, 'train')
def test_reducemin_training_negative_axis(op_tester):
"""Test the above but with equivalent negative axes.
Args:
op_tester (OpTester): OpTEster object, see op_tester.py
Returns:
Nothing
"""
data = np.random.rand(2, 5, 3).astype(np.float32)
axes_list = [[0], [-1], [-2], [0, -1], [0, -2], [1, -3], [0, -1, -2],
USE_DEFAULT_AXES]
keepdims_list = [False, True]
def init_builder(builder):
result = []
axes_reduce = []
for axes, keepdims in itertools.product(axes_list, keepdims_list):
tensor = builder.addInputTensor(data)
if axes is USE_DEFAULT_AXES:
out = builder.aiOnnx.reducemin(
[tensor],
keepdims=keepdims,
debugContext="test_reducemin_default_{0}".format(keepdims))
else:
out = builder.aiOnnx.reducemin(
[tensor],
axes=axes,
keepdims=keepdims,
debugContext="test_reducemin_{0}_{1}".format(
axes, keepdims))
result.append(out)
result.append(popart.reservedGradientPrefix() + tensor)
axes_len = len(axes) if axes is not USE_DEFAULT_AXES else 3
axes_reduce.append(range(3 - (0 if keepdims else axes_len)))
sum = builder.aiOnnx.sum([
builder.aiOnnx.reducesum([r],
axes=axes,
keepdims=False,
debugContext="test_reducesum_all")
for r, axes in zip(result[0::2], axes_reduce)
],
debugContext="test_sum")
reshaped_sum = builder.aiOnnx.unsqueeze([sum],
axes=[0],
debugContext="test_reshape")
builder.addOutputTensor(reshaped_sum)
result = [
reshaped_sum,
popart.reservedGradientPrefix() + reshaped_sum
] + result
return result
def reference(ref_data):
result = []
for axes, keepdims in itertools.product(axes_list, keepdims_list):
tensor = torch.tensor(data, requires_grad=True)
out = tensor
sorted_axes = axes if axes is not USE_DEFAULT_AXES else [0, 1, 2]
print(sorted_axes)
# Have to manually norm the axes as torch doesn't like -1 * rank
# axes.
for i in range(len(sorted_axes)):
axis = sorted_axes[i]
if axis < 0:
sorted_axes[i] = len(out.size()) + axis
sorted_axes.sort(reverse=True)
print("\t", sorted_axes)
for dim in sorted_axes:
out = torch.min(out, dim=dim, keepdim=keepdims)[0]
result.append(out)
result.append(tensor)
sum = torch.unsqueeze(torch.sum(
torch.stack([torch.sum(r) for r in result[0::2]])),
dim=0)
d__o = ref_data.getOutputTensorGrad(0)
sum.backward(torch.tensor(d__o))
result[1::2] = [r.grad for r in result[1::2]]
result = [sum, sum.grad] + result
return result
op_tester.setPatterns(['OpToIdentity'], enableRuntimeAsserts=False)
op_tester.run(init_builder, reference, 'train')
def test_call_grad_1(op_tester):
shape = [4, 4]
d0 = np.random.normal(size=shape).astype(np.float32)
d1 = np.random.normal(size=shape).astype(np.float32)
d2 = np.random.normal(size=shape).astype(np.float32)
def get_init_builder(input_tensor_method):
def init_builder(builder):
i0 = builder.addInputTensor(d0)
i1 = builder.addInputTensor(d1)
i2 = builder.addInputTensor(d2)
subgraph_builder = builder.createSubgraphBuilder()
if input_tensor_method == "untyped":
sgi0 = subgraph_builder.addUntypedInputTensor()
sgi1 = subgraph_builder.addUntypedInputTensor()
elif input_tensor_method == "with_info":
info = popart.TensorInfo("FLOAT", shape)
sgi0 = subgraph_builder.addInputTensor(info)
sgi1 = subgraph_builder.addInputTensor(info)
elif input_tensor_method == "from_higher_scope":
subgraph_builder.addInputTensorFromParentGraph(i0)
subgraph_builder.addInputTensorFromParentGraph(i1)
if input_tensor_method == "from_higher_scope":
subgraph_builder.addOutputTensor(
subgraph_builder.aiOnnx.matmul([i0, i1]))
else:
subgraph_builder.addOutputTensor(
subgraph_builder.aiOnnx.matmul([sgi0, sgi1]))
act = builder.aiGraphcore.call([i0, i1], 1, subgraph_builder)[0]
out = builder.aiGraphcore.call([act, i2], 1, subgraph_builder)[0]
builder.addOutputTensor(out)
return [
out,
popart.reservedGradientPrefix() + out,
popart.reservedGradientPrefix() + i0,
popart.reservedGradientPrefix() + i1,
popart.reservedGradientPrefix() + i2
]
return init_builder
def reference(ref_data):
t0 = torch.tensor(d0, requires_grad=True)
t1 = torch.tensor(d1, requires_grad=True)
t2 = torch.tensor(d2, requires_grad=True)
o = torch.chain_matmul(t0, t1, t2)
d__o = ref_data.getOutputTensorGrad(0)
o.backward(torch.tensor(d__o))
return [o, d__o, t0.grad, t1.grad, t2.grad]
op_tester.setPatterns(popart.PatternsLevel.Default,
enableRuntimeAsserts=False)
op_tester.run(get_init_builder("untyped"), reference, 'train')
op_tester.run(get_init_builder("with_info"), reference, 'train')
op_tester.run(get_init_builder("from_higher_scope"), reference, 'train')