def init_builder(builder): i1 = builder.addInputTensor(d1) o = builder.aiOnnx.sign([i1], "test_sign") builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + o, popart.reservedGradientPrefix() + i1, ]
def init_builder(builder): i1 = builder.addInputTensor(d1) o = builder.aiGraphcore.reshape(i1, shape=d2a) builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + i1, popart.reservedGradientPrefix() + o, ]
def _get_ir(pingpong_enabled, virtualgraph_enabled, pipeline_enabled): dsize = 10 builder = popart.Builder() ip = builder.addInputTensor(popart.TensorInfo("FLOAT", [dsize, dsize])) anchorIds = [] anchorIds.append(popart.reservedGradientPrefix() + ip) def add_layer(in_id): w = builder.addInitializedInputTensor( np.ones([dsize, dsize], np.float32)) matmul_id = builder.aiOnnx.matmul([in_id, w]) return matmul_id x = ip for i in range(3): w = builder.addInitializedInputTensor( np.ones([dsize, dsize], np.float32)) x = builder.aiOnnx.matmul([x, w]) if pingpong_enabled: builder.pingPongPhase(x, i) if virtualgraph_enabled: builder.virtualGraph(x, i) if pipeline_enabled: builder.pipelineStage(x, i) anchorIds.append(popart.reservedGradientPrefix() + x) out = builder.aiGraphcore.identityloss([x]) if virtualgraph_enabled: builder.virtualGraph(out, 3) device = tu.create_test_device() dfAnchors = {} for anchorId in anchorIds: dfAnchors.update({anchorId: popart.AnchorReturnType("All")}) opts = popart.SessionOptions() # disable outlining to make the ir easier to parse opts.enableOutlining = False proto = builder.getModelProto() session = popart.TrainingSession(fnModel=proto, dataFlow=popart.DataFlow(1, dfAnchors), optimizer=popart.ConstSGD(0.1), loss=out, patterns=popart.Patterns( popart.PatternsLevel.All), userOptions=opts, deviceInfo=device) ir = json.loads(session._serializeIr(popart.IrSerializationFormat.JSON)) return ir
def run(model_file_name, enableOutlining): dsize = 10 ratio = 0.5 builder = popart.Builder() ip = builder.addInputTensor(popart.TensorInfo("FLOAT", [dsize, dsize])) d__ip = popart.reservedGradientPrefix() + ip def add_layer(in_id): w = builder.addInitializedInputTensor( np.ones([dsize, dsize], np.float32)) matmul_id = builder.aiOnnx.matmul([in_id, w]) return matmul_id m1 = add_layer(ip) m2 = add_layer(m1) m3 = add_layer(m2) anchorIds = [] for i in (ip, m1, m2, m3): anchorIds.append(popart.reservedGradientPrefix() + i) out = builder.aiGraphcore.identityloss([m3]) builder.addOutputTensor(out) device = tu.create_test_device() dfAnchors = {} for anchorId in anchorIds: dfAnchors.update({anchorId: popart.AnchorReturnType("All")}) opts = popart.SessionOptions() opts.enableOutlining = enableOutlining opts.separateCallOpPdfs = False opts.subgraphCopyingStrategy = subgraphCopyingStrategy proto = builder.getModelProto() session = popart.TrainingSession( fnModel=proto, dataFlow=popart.DataFlow(1, dfAnchors), optimizer=popart.ConstSGD(0.1), loss=out, patterns=popart.Patterns(popart.PatternsLevel.All), userOptions=opts, deviceInfo=device) session.prepareDevice() session.weightsFromHost() anchors = session.initAnchorArrays() ip_data = np.ones((dsize, dsize), dtype=np.float32) stepio = popart.PyStepIO({ip: ip_data}, anchors) session.run(stepio) session.modelToHost(str(tmpdir / model_file_name))
def init_builder(builder): D = builder.addInputTensor(src.numpy()) I = builder.addInputTensor(index.numpy().astype(np.uint32)) out = builder.aiGraphcore.scatterreduce([D, I], axis_size=axsz) builder.addOutputTensor(out) return [ out, popart.reservedGradientPrefix() + D, popart.reservedGradientPrefix() + out ]
def init_builder(builder): i0 = builder.addInputTensor(x) i1 = builder.aiOnnxOpset11.constant(axis) o = builder.aiOnnxOpset11.cumsum([i0, i1], reverse=1) builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + i0, popart.reservedGradientPrefix() + o, ]
def init_builder(builder): i1 = builder.addInputTensor(d1) c = builder.aiOnnx.constant(d2) o = builder.aiOnnx.tile([i1, c]) builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + i1, popart.reservedGradientPrefix() + o ]
def init_builder(builder): i1 = builder.addInputTensor(d1) c = builder.aiOnnx.cast([i1], builderDstType) # Add an op that produces a gradient so we can test CastGrad properly o = builder.aiOnnx.reducesum([c]) builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + i1, popart.reservedGradientPrefix() + o ]
def init_builder(builder): i1 = builder.addInputTensor(d1) i2 = builder.addInputTensor(d2) o = builder.aiOnnx.pow([i1, i2]) builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + i1, popart.reservedGradientPrefix() + i2, popart.reservedGradientPrefix() + o ]
def init_builder(builder): P = builder.addInitializedInputTensor(data) T = builder.addInputTensor(target.astype(np.int32)) logP = builder.aiOnnx.logsoftmax([P], axis=1) nll = builder.aiGraphcore.nllloss([logP, T], inputIsLogProbability=1) builder.addOutputTensor(nll) return [ nll, popart.reservedGradientPrefix() + P, popart.reservedGradientPrefix() + nll, ]
def init_builder(builder): i1 = builder.addInputTensor(d1) o = builder.aiOnnxOpset11.depthtospace([i1], blocksize=blocks, mode="DCR") builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + i1, popart.reservedGradientPrefix() + o, ]
def init_builder(builder): i1 = builder.addInputTensor(indices) i2 = builder.aiOnnx.constant(depth) # depth has to be a constant i3 = builder.addInputTensor(values) o = builder.aiOnnx.onehot([i1, i2, i3], 0, "test_onehot") builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + o, popart.reservedGradientPrefix() + i3 ]
def init_builder(builder): i1 = builder.addInputTensor(data) i2 = builder.addInputTensor(indices) i3 = builder.addInputTensor(updates) o = builder.aiOnnx.scatter([i1, i2, i3], axis) builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + i1, popart.reservedGradientPrefix() + i3 ]
def init_builder(builder): i1 = builder.addInputTensor(input_y) i2 = builder.addInputTensor(input_x) o = builder.aiGraphcore.atan2([i1, i2]) builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + i1, popart.reservedGradientPrefix() + i2, popart.reservedGradientPrefix() + o ]
def init_builder(builder): tensor = builder.addInputTensor(data) out = builder.aiGraphcore.swish( [tensor], debugContext='test_swish_grad', ) builder.addOutputTensor(out) return [ out, popart.reservedGradientPrefix() + out, popart.reservedGradientPrefix() + tensor, ]
def init_builder(builder): d = builder.addInputTensor(data) x = builder.addInputTensor(x_data) s = builder.aiOnnx.constant(scales) o = builder.aiOnnx.resize([d, s]) o = builder.aiOnnx.mul([o, x]) builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + d, popart.reservedGradientPrefix() + o, ]
def init_builder(builder): i1 = builder.addInputTensor(d1) axes = builder.aiOnnx.constant(axesV) starts = builder.aiOnnx.constant(startsV) ends = builder.aiOnnx.constant(endsV) o = builder.aiOnnx.slice([i1, starts, ends, axes]) builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + i1, popart.reservedGradientPrefix() + o ]
def init_builder(builder): i1 = builder.addInputTensor(data) i2 = builder.addInputTensor(one_place) s = builder.aiOnnx.softmax([i1], axis) o = builder.aiOnnx.mul([s, i2]) builder.addOutputTensor(o) return [ o, s, popart.reservedGradientPrefix() + i1, popart.reservedGradientPrefix() + o ]
def init_builder(builder): i1 = builder.addInputTensor(A) i2 = builder.addInputTensor(B) i3 = builder.addInputTensor(C) o = builder.aiOnnx.gemm([i1, i2, i3], alpha, beta, transA, transB) builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + i1, popart.reservedGradientPrefix() + i2, popart.reservedGradientPrefix() + i3, popart.reservedGradientPrefix() + o ]
def init_builder(builder): i = builder.addInputTensor(data) a, b = builder.aiOnnx.split([i], 2) o = builder.aiOnnx.sum([a, b]) builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + i, popart.reservedGradientPrefix() + o ]
def init_builder(builder): i1 = builder.addInputTensor(input_data) o = builder.aiOnnx.softplus([i1]) builder.addOutputTensor(o) op_tester.setPatterns(['InPlace'], enableRuntimeAsserts=False) # Set the result to # ['Softplus:0', 'Gradient___input', 'Gradient___Softplus:0'] result = [ o, popart.reservedGradientPrefix() + i1, popart.reservedGradientPrefix() + o ] return result
def run_test(): builder = popart.Builder() lhs = builder.addInputTensor(popart.TensorInfo("FLOAT", lhs_shape), "lhs") rhs = builder.addInputTensor(popart.TensorInfo("FLOAT", rhs_shape), "rhs") z = builder.addInputTensor(popart.TensorInfo("FLOAT", [2]), "zero") t1 = builder.aiOnnx.matmul([lhs, rhs]) o = builder.aiOnnx.add([z, t1]) o = builder.aiGraphcore.identityloss([o]) proto = builder.getModelProto() dataFlow = popart.DataFlow( 1, { o: popart.AnchorReturnType("All"), popart.reservedGradientPrefix() + lhs: popart.AnchorReturnType("All"), popart.reservedGradientPrefix() + rhs: popart.AnchorReturnType("All"), }) opts = popart.SessionOptions() opts.reportOptions = {"showExecutionSteps": "true"} pat = popart.Patterns(popart.PatternsLevel.Default) session = popart.TrainingSession( fnModel=proto, dataFlow=dataFlow, userOptions=opts, loss=o, optimizer=popart.ConstSGD(0.01), patterns=pat, deviceInfo=tu.create_test_device(opts={"compileIPUCode": False})) session.prepareDevice() anchors = session.initAnchorArrays() inputs = {lhs: lhs_data, rhs: rhs_data, z: zero_data} stepio = popart.PyStepIO(inputs, anchors) session.run(stepio) return anchors[o]
def init_builder(builder): i1 = builder.addInputTensor(input_data) o = builder.aiOnnx.hardsigmoid([i1], alpha=alpha, beta=beta) builder.addOutputTensor(o) result = [o] if builder_settings is 'InPlace': op_tester.setPatterns(['InPlace'], enableRuntimeAsserts=False) elif builder_settings is 'backward': result = [ o, popart.reservedGradientPrefix() + i1, popart.reservedGradientPrefix() + o ] return result
def init_builder(builder): i1 = builder.addInputTensor(A) i2 = builder.addInitializedInputTensor(B) i3 = builder.addInitializedInputTensor(C) o = builder.aiOnnx.gemm([i1, i2, i3], alpha, beta, transA, transB) builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + i2, i2, popart.reservedGradientPrefix() + i3, i3, "scaledLearningRate0___default___FLOAT", "weightDecayScaleFactor0___default___FLOAT" ]
def init_builder(builder): i1 = builder.addInputTensor(d1) (o, ) = builder.aiOnnx.maxpool([i1], num_outputs=1, kernel_shape=[2, 2], pads=[0, 0, 0, 0], storage_order=0, strides=[2, 2]) builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + i1, popart.reservedGradientPrefix() + o ]
def init_builder(builder): i1 = builder.addInputTensor(d1) c = builder.aiOnnx.constant(d2) o = builder.aiOnnx.expand([i1, c]) if inplace: o_identity = builder.aiOnnx.identity([o]) else: o_identity = o builder.addOutputTensor(o_identity) return [ o_identity, popart.reservedGradientPrefix() + o, popart.reservedGradientPrefix() + i1 ]
def init_builder(builder): i1 = builder.addInputTensor(input_data) o = builder.aiOnnx.elu([i1], alpha=alpha) builder.addOutputTensor(o) result = [o] if builder_settings is 'InPlace': op_tester.patterns = ['InPlace'] elif builder_settings is 'backward': result = [ o, popart.reservedGradientPrefix() + i1, popart.reservedGradientPrefix() + o ] return result
def init_builder(builder): i1 = builder.addInputTensor(input_data) if (alpha == None): o = builder.aiOnnx.leakyrelu([i1]) else: o = builder.aiOnnx.leakyrelu([i1], alpha=alpha) builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + i1, popart.reservedGradientPrefix() + o ]
def init_builder(builder): tData = builder.addInputTensor(data, data_id) tIW = builder.addInitializedInputTensor(input_weights, input_weights_id) tOW = builder.addInitializedInputTensor(output_weights, output_weights_id) tBiases = builder.addInitializedInputTensor(biases, biases_id) tInitH = builder.addInputTensor(initial_h, init_h_id) tInitC = builder.addInputTensor(initial_c, init_c_id) def reshape_weights(w): ws = builder.aiOnnx.split([w], 4, 1, [hidden_size] * 4) ws = [builder.aiOnnx.transpose([i], [0, 2, 1]) for i in ws] ws = builder.aiOnnx.concat([ws[i] for i in (2, 0, 3, 1)], 0) return ws tIW = reshape_weights(tIW) tOW = reshape_weights(tOW) # NB shape inference is not yet possible with aiOnnx.split tWeights = builder.aiOnnx.concat([tIW, tOW], 1) tBiases = builder.aiOnnx.split([tBiases], 8, 1, [hidden_size] * 8) tBiases0 = builder.aiOnnx.concat( [tBiases[i] for i in (2, 0, 3, 1)], 0) tBiases1 = builder.aiOnnx.concat( [tBiases[i] for i in (6, 4, 7, 5)], 0) tBiases = builder.aiOnnx.add([tBiases0, tBiases1]) tInitState = builder.aiOnnx.concat([tInitH, tInitC], 0) input_ids = [tData, tWeights, tBiases, tInitState] out, cell_state = builder.aiGraphcore.lstm(input_ids) assert builder.getTensorDtypeString(out) == "float32" assert builder.getTensorDtypeString(cell_state) == "float32" if sum_outputs: out = builder.aiOnnx.add([out, cell_state], "sum_out_and_cell_state") loss = builder.aiGraphcore.identityloss([out]) return [ loss, popart.reservedGradientPrefix() + data_id, popart.reservedGradientPrefix() + input_weights_id, popart.reservedGradientPrefix() + output_weights_id, popart.reservedGradientPrefix() + biases_id, ]
def init_builder_manual_padding(builder): i1 = builder.addInputTensor(d1) (o, ) = builder.aiOnnxOpset10.maxpool([i1], num_outputs=1, kernel_shape=[3, 3], ceil_mode=0, pads=[0, 0, 1, 1], storage_order=0, strides=[2, 2]) builder.addOutputTensor(o) return [ o, popart.reservedGradientPrefix() + i1, popart.reservedGradientPrefix() + o ]