def test_ipu_copy_aca1(): popart.getLogger().setLevel("TRACE") builder = popart.Builder() i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1])) i2 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1])) o1 = builder.aiOnnx.add([i1, i2]) o2 = builder.aiOnnx.add([i1, i2]) o = builder.aiOnnx.add([o1, o2]) builder.addOutputTensor(o) builder.virtualGraph(o1, 0) builder.virtualGraph(o2, 0) builder.virtualGraph(o, 10) # << Invalid virtual graph proto = builder.getModelProto() dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")}) opts = popart.SessionOptions() opts.virtualGraphMode = popart.VirtualGraphMode.Manual s = popart.InferenceSession(fnModel=proto, dataFlow=dataFlow, userOptions=opts, deviceInfo=tu.create_test_device(numIpus=3)) with pytest.raises(popart.popart_exception) as e_info: s.prepareDevice() assert (("inputs=[{}, {}], outputs=[{}]) " + "has been assigned to an invalid virtual graph 10").format( o1 + "_c10", o2 + "_c10", o) in e_info.value.args[0])
def test_engine_options_passed_to_engine(tmpdir): popart.getLogger().setLevel("DEBUG") builder = popart.Builder() shape = popart.TensorInfo("FLOAT", [1, 2, 32, 32]) i1 = builder.addInputTensor(shape) i2 = builder.addInputTensor(shape) o = builder.aiOnnx.add([i1, i2]) builder.addOutputTensor(o) proto = builder.getModelProto() dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")}) opts = popart.SessionOptions() opts.engineOptions = {'option': 'value'} session = popart.InferenceSession(fnModel=proto, dataFlow=dataFlow, userOptions=opts, deviceInfo=tu.create_test_device()) session.initAnchorArrays() with pytest.raises(popart.poplar_exception) as e_info: session.prepareDevice() assert (e_info.value.args[0].endswith("Unrecognised option 'option'"))
def test_ipu_copy_bca5(): popart.getLogger().setLevel("TRACE") builder = popart.Builder() constData = np.random.rand(2, 2).astype(np.float32) c1 = builder.aiOnnx.constant(constData, "constShapeData") i2 = builder.addInputTensor(popart.TensorInfo("FLOAT", [2, 2])) o1 = builder.aiOnnx.add([c1, i2]) o2 = builder.aiOnnx.add([c1, i2]) t1 = builder.aiOnnx.transpose([c1], []) o3 = builder.aiOnnx.add([o1, o2]) o = builder.aiOnnx.add([o3, t1]) builder.addOutputTensor(o) builder.virtualGraph(o1, 0) builder.virtualGraph(o2, 2) builder.virtualGraph(t1, 2) builder.virtualGraph(o3, 1) builder.virtualGraph(o, 1) proto = builder.getModelProto() dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")}) opts = popart.SessionOptions() opts.virtualGraphMode = popart.VirtualGraphMode.Manual s = popart.InferenceSession(fnModel=proto, dataFlow=dataFlow, userOptions=opts, deviceInfo=tu.create_test_device(numIpus=3)) s.prepareDevice()
def test_virtual_graph_multi_output(container): popart.getLogger().setLevel("TRACE") builder = popart.Builder() i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [4, 1])) o1, o2, o3, o4 = builder.aiOnnx.split([i1], 4) o5 = builder.aiOnnx.add([o1, o2]) o6 = builder.aiOnnx.add([o3, o4]) o = builder.aiOnnx.add([o5, o6]) builder.addOutputTensor(o) builder.virtualGraph(container([o1, o2, o3, o4]), 1) builder.virtualGraph(o5, 0) builder.virtualGraph(o6, 0) builder.virtualGraph(o, 0) # Check the virtual graph was set correctly assert builder.getVirtualGraph(container([o1, o2, o3, o4])) == 1 proto = builder.getModelProto() dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")}) opts = popart.SessionOptions() opts.virtualGraphMode = popart.VirtualGraphMode.Manual s = popart.InferenceSession(fnModel=proto, dataFlow=dataFlow, userOptions=opts, deviceInfo=tu.create_test_device(numIpus=2)) s.prepareDevice()
def test_copy_to_op_with_duplicate_inputs(): popart.getLogger().setLevel("TRACE") builder = popart.Builder() i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1])) o1 = builder.aiOnnx.add([i1, i1]) builder.virtualGraph(o1, 0) o2 = builder.aiOnnx.mul([i1, i1]) builder.virtualGraph(o2, 1) o3 = builder.aiOnnx.add([o1, o2]) builder.virtualGraph(o3, 1) o = o3 builder.addOutputTensor(o) proto = builder.getModelProto() dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")}) opts = popart.SessionOptions() opts.virtualGraphMode = popart.VirtualGraphMode.Manual s = popart.InferenceSession(fnModel=proto, dataFlow=dataFlow, userOptions=opts, deviceInfo=tu.create_test_device(numIpus=3)) s.prepareDevice()
def test_mixed_virtual_graph(): popart.getLogger().setLevel("TRACE") builder = popart.Builder() i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1])) i2 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1])) o1 = builder.aiOnnx.add([i1, i2]) o2 = builder.aiOnnx.add([i1, i2]) o = builder.aiOnnx.add([o1, o2]) builder.addOutputTensor(o) builder.virtualGraph(o1, 0) builder.virtualGraph(o2, 1) proto = builder.getModelProto() dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")}) opts = popart.SessionOptions() opts.virtualGraphMode = popart.VirtualGraphMode.Manual with pytest.raises(popart.popart_exception) as e_info: popart.InferenceSession(fnModel=proto, dataFlow=dataFlow, userOptions=opts, deviceInfo=tu.create_test_device()) assert (e_info.value.args[0].startswith("Either all"))
def test_virtual_graph(): popart.getLogger().setLevel("TRACE") builder = popart.Builder() i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1])) i2 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1])) i3 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1])) i4 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1])) o1 = builder.aiOnnx.add([i1, i2]) o2 = builder.aiOnnx.add([i3, i4]) o = builder.aiOnnx.add([o1, o2]) builder.addOutputTensor(o) builder.virtualGraph(o1, 1) builder.virtualGraph(o2, 1) builder.virtualGraph(o, 1) proto = builder.getModelProto() dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")}) opts = popart.SessionOptions() opts.virtualGraphMode = popart.VirtualGraphMode.Manual s = popart.InferenceSession(fnModel=proto, dataFlow=dataFlow, userOptions=opts, deviceInfo=tu.create_test_device(numIpus=2)) s.prepareDevice() pass
def test_auto_virtual_graph_not_enough_splits(): ipus = 4 popart.getLogger().setLevel("TRACE") builder = popart.Builder() input_shape = [1, 64] input = builder.addInputTensor(popart.TensorInfo("FLOAT16", input_shape)) x = input for i in range(2): w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16)) x = builder.aiOnnx.matmul([x, w]) output = x builder.addOutputTensor(output) proto = builder.getModelProto() dataFlow = popart.DataFlow(1, {output: popart.AnchorReturnType("Final")}) opts = popart.SessionOptions() opts.virtualGraphMode = popart.VirtualGraphMode.Auto device = tu.create_test_device(numIpus=ipus) with pytest.raises(popart.popart_exception) as e_info: popart.InferenceSession(fnModel=proto, dataFlow=dataFlow, userOptions=opts, deviceInfo=device) assert (e_info.value.args[0].startswith( "[AutoVirtualGraph] Couldn't find enough splits"))
def test_auto_virtual_graph_inf_many(): ipus = 4 popart.getLogger().setLevel("TRACE") builder = popart.Builder() input_shape = [1, 64] input = builder.addInputTensor(popart.TensorInfo("FLOAT16", input_shape)) x = input for i in range(16): w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16)) x = builder.aiOnnx.matmul([x, w]) output = x builder.addOutputTensor(output) proto = builder.getModelProto() dataFlow = popart.DataFlow(1, {output: popart.AnchorReturnType("Final")}) opts = popart.SessionOptions() opts.virtualGraphMode = popart.VirtualGraphMode.Auto device = tu.create_test_device(numIpus=ipus) popart.InferenceSession(fnModel=proto, dataFlow=dataFlow, userOptions=opts, deviceInfo=device)
def test_set_random_seed_error(): popart.getLogger().setLevel("TRACE") builder = popart.Builder() i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [10])) [o] = builder.aiOnnx.dropout([i1], num_outputs=1, ratio=0.3) loss = builder.aiGraphcore.identityloss([o]) proto = builder.getModelProto() dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")}) s = popart.TrainingSession(fnModel=proto, dataFlow=dataFlow, optimizer=popart.ConstSGD(0.1), loss=loss, userOptions=popart.SessionOptions(), deviceInfo=tu.create_test_device(numIpus=2)) with pytest.raises(popart.popart_exception) as e_info: s.setRandomSeed(0) msg = e_info.value.args[0] assert msg == ("Devicex::prepare() must be called before " "Devicex::setRandomSeedFromHost(uint64_t) is called.")
def test_model_caching(tmp_path, capfd): # Check the log output to see if an engine was compiled, # or if a cached engine was used. def loaded_saved_executable(): _, stderr = capfd.readouterr() startedEngineCompilation = False loadedPoplarExecutable = False for line in stderr.splitlines(): if 'Starting compilation' in line: startedEngineCompilation = True elif 'Loading serialized PopART executable' in line: loadedPoplarExecutable = True assert startedEngineCompilation != loadedPoplarExecutable return startedEngineCompilation popart.getLogger().setLevel('DEBUG') cache_path = str(tmp_path / 'model_caching') result0 = build_and_run(cache_path) assert loaded_saved_executable() is True result1 = build_and_run(cache_path) assert loaded_saved_executable() is False assert result0 == result1 shutil.rmtree(cache_path) build_and_run(cache_path) assert loaded_saved_executable() is True
def test_average_pool_with_count_include_pad(op_tester): popart.getLogger().setLevel("TRACE") builder = popart.Builder() i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1, 1, 14, 14])) o = builder.aiOnnx.averagepool([i1], kernel_shape=[3, 3], count_include_pad=1, pads=[0, 0, 0, 0], strides=[2, 2]) builder.addOutputTensor(o) builder.removeNodeAttribute("count_include_pad", set([o])) builder.addNodeAttribute("count_include_pad", 1, set([o])) dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")}) optimizer = popart.ConstSGD(0.01) loss = builder.aiGraphcore.l1loss([o], 0.1) proto = builder.getModelProto() opts = popart.SessionOptions() with pytest.raises(popart.popart_exception) as e_info: popart.TrainingSession(fnModel=proto, dataFlow=dataFlow, loss=loss, optimizer=optimizer, userOptions=opts, deviceInfo=tu.create_test_device()) assert (e_info.value.args[0].startswith( "`count_include_pad` is not supported"))
def __init__(self, dataset, batch_size=1, shuffle=False, sampler=None, batch_sampler=None, num_workers=0, collate_fn=default_collate, drop_last=False, tensor_type=None, timeout=0, worker_init_fn=None, log_statisics=False): self.dataset = dataset self.batch_size = batch_size self.num_workers = num_workers self.collate_fn = partial(collate_fn, tensor_type=tensor_type) self.drop_last = drop_last self.timeout = timeout self.worker_init_fn = worker_init_fn self.log_statisics = log_statisics popart.getLogger().info( "DataLoader created batchsize:{} num_workers:{} " "shuffle:{} tensor_type:{}".format(batch_size, num_workers, shuffle, tensor_type)) if timeout < 0: raise ValueError('timeout option should be non-negative') if batch_sampler is not None: if batch_size > 1 or shuffle or sampler is not None or drop_last: raise ValueError('batch_sampler option is mutually exclusive ' 'with batch_size, shuffle, sampler, and ' 'drop_last') self.batch_size = None self.drop_last = None if sampler is not None and shuffle: raise ValueError('sampler option is mutually exclusive with ' 'shuffle') if self.num_workers < 0: raise ValueError('num_workers option cannot be negative; ' 'use num_workers=0 to disable multiprocessing.') if batch_sampler is None: if sampler is None: if shuffle: sampler = torch.utils.data.RandomSampler(dataset) else: sampler = torch.utils.data.SequentialSampler(dataset) batch_sampler = torch.utils.data.BatchSampler( sampler, batch_size, drop_last) self.sampler = sampler self.batch_sampler = batch_sampler self.dataloaderiter = None
def check_loglevel(level): popart.getLogger().setLevel(level) builder = popart.Builder() i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1, 2, 3])) o = builder.aiOnnx.abs([i1]) _, err = capfd.readouterr() return err
def test_constants_preserved(): popart.getLogger().setLevel("TRACE") # Check that `session.modelToHost` can be called when using a # model with a constant node, without throwing an exceptions builder = popart.Builder() i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [2, 2])) i2 = builder.addInputTensor(popart.TensorInfo("FLOAT", [2, 2])) c = builder.aiOnnx.constant(np.array([[1, 2], [3, 4]], dtype=np.float32)) o1 = builder.aiOnnx.add([i1, i2]) o2 = builder.aiOnnx.add([o1, c]) loss = builder.aiGraphcore.identityloss([o2]) proto = builder.getModelProto() anchors = {o2: popart.AnchorReturnType("All")} dataFlow = popart.DataFlow(1, anchors) optimizer = popart.ConstSGD(0.01) opts = popart.SessionOptions() session = popart.TrainingSession(fnModel=proto, dataFlow=dataFlow, userOptions=opts, loss=loss, optimizer=optimizer, deviceInfo=tu.create_test_device()) anchorArrays = session.initAnchorArrays() session.prepareDevice() inputs = { i1: np.array([[2, 2], [2, 2]]).astype(np.float32), i2: np.array([[4, 4], [4, 4]]).astype(np.float32), } pystepio = popart.PyStepIO(inputs, anchorArrays) session.run(pystepio) session.modelToHost('session_proto.onnx') # models should be the same after training # as there are no trainable parameters with open('session_proto.onnx', 'rb') as f: session_proto = f.read() assert proto == session_proto # confirm that the output is correct. See T6186, which this tests assert (np.sum(np.abs(anchorArrays[o2] - np.array([[7, 8], [9, 10]]))) < 1e-8)
def test_sgd_with_float16_model(): popart.getLogger().setLevel("TRACE") input1 = np.zeros((2, 2, 4, 4), dtype=np.float16) input2 = np.zeros((2, 2, 3, 3), dtype=np.float16) input3 = np.zeros((2, 2, 3, 3), dtype=np.float16) builder = popart.Builder() inid1 = builder.addInputTensor(popart.TensorInfo(input1)) inid2 = builder.addInitializedInputTensor(input2) inid3 = builder.addInitializedInputTensor(input2) c1 = builder.aiOnnx.conv([inid1, inid2], dilations=[1, 1], pads=[1, 1, 1, 1], strides=[1, 1]) c2 = builder.aiOnnx.conv([c1, inid3], dilations=[1, 1], pads=[1, 1, 1, 1], strides=[1, 1]) # Reduce to scalar out = builder.aiGraphcore.identityloss([c2]) proto = builder.getModelProto() optimizer = popart.SGD({ "defaultLearningRate": (0.1, False), "defaultWeightDecay": (0.1, False), "lossScaling": (1000, False) }) anchorNames = { popart.reservedGradientPrefix() + inid1: popart.AnchorReturnType("All"), } opts = popart.SessionOptions() session = popart.TrainingSession( fnModel=proto, dataFlow=popart.DataFlow(1, anchorNames), loss=out, optimizer=optimizer, deviceInfo=tu.create_test_device(opts={"compileIPUCode": False})) session.prepareDevice() session.weightsFromHost() anchorArrays = session.initAnchorArrays() stepio = popart.PyStepIO({inid1: input1}, anchorArrays) session.run(stepio)
def prepare( cls, model, # type: ModelProto device='IPU', # type: Text **kwargs # type: Any ): # type: (...) -> Optional[onnx.backend.base.BackendRep] super(IpuBackend, cls).prepare(model, device, **kwargs) # test shape inference model = onnx.shape_inference.infer_shapes(model) value_infos = { vi.name: vi for vi in itertools.chain(model.graph.value_info, model.graph.output) } # if do_enforce_test_coverage_whitelist(model): # for node in model.graph.node: # for i, output in enumerate(node.output): # if node.op_type == 'Dropout' and i != 0: # continue # assert output in value_infos # tt = value_infos[output].type.tensor_type # assert tt.elem_type != TensorProto.UNDEFINED # for dim in tt.shape.dim: # assert dim.WhichOneof('value') == 'dim_value' popart.getLogger().setLevel("DEBUG") opts = popart.SessionOptions() anchors = {} for output in model.graph.output: anchors[output.name] = popart.AnchorReturnType("All") session = popart.InferenceSession( fnModel=model.SerializeToString(), dataFlow=popart.DataFlow(1, anchors), deviceInfo=popart.DeviceManager().createCpuDevice(), # deviceInfo=popart.DeviceManager().createIpuModelDevice({}), userOptions=opts) session.prepareDevice() context = Context(session, model) return context
def test_cache_environment_variable(tmp_path, capfd): """ Test chaching enabled via env POPART_CACHE_DIR """ popart.getLogger().setLevel('DEBUG') os.environ['POPART_CACHE_DIR'] = str(tmp_path / 'saved_graph') opts = popart.SessionOptions() run_session(2, opts) assert loaded_saved_executable(capfd) is False # Check engine caching works for two identical sessions. run_session(2, opts) assert loaded_saved_executable(capfd) is True del os.environ['POPART_CACHE_DIR']
def test_auto_virtual_graph_subgraphs_2(): ipus = 2 popart.getLogger().setLevel("TRACE") builder = popart.Builder() input_shape = [1, 64] input1 = builder.addInputTensor(popart.TensorInfo("FLOAT16", input_shape)) input2 = builder.addInputTensor(popart.TensorInfo("FLOAT16", input_shape)) # Subgraph 0 w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16)) x0 = builder.aiOnnx.matmul([input1, w]) w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16)) x0 = builder.aiOnnx.matmul([x0, w]) # Subgraph 1 w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16)) x1 = builder.aiOnnx.matmul([input2, w]) # Subgraph 2 x2 = builder.aiOnnx.add([x0, x1]) w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16)) x2 = builder.aiOnnx.matmul([x2, w]) output = x2 builder.addOutputTensor(output) # Desired split is: # ipu1: 0. ipu2: 1,2 proto = builder.getModelProto() dataFlow = popart.DataFlow(1, {output: popart.AnchorReturnType("Final")}) opts = popart.SessionOptions() opts.virtualGraphMode = popart.VirtualGraphMode.Auto device = tu.create_test_device(numIpus=ipus) popart.InferenceSession(fnModel=proto, dataFlow=dataFlow, userOptions=opts, deviceInfo=device)
def test_auto_virtual_graph_subgraphs_1(): ipus = 1 popart.getLogger().setLevel("TRACE") builder = popart.Builder() input_shape = [1, 64] input1 = builder.addInputTensor(popart.TensorInfo("FLOAT16", input_shape)) input2 = builder.addInputTensor(popart.TensorInfo("FLOAT16", input_shape)) # Subgraph 0 w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16)) x0 = builder.aiOnnx.matmul([input1, w]) w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16)) x0 = builder.aiOnnx.matmul([x0, w]) # Subgraph 1 w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16)) x1 = builder.aiOnnx.matmul([input2, w]) output = x1 builder.addOutputTensor(output) # Only possible split is: # ipu1: 0, 1 proto = builder.getModelProto() dataFlow = popart.DataFlow(1, {output: popart.AnchorReturnType("Final")}) opts = popart.SessionOptions() opts.virtualGraphMode = popart.VirtualGraphMode.Auto device = tu.create_test_device(numIpus=ipus) session = popart.InferenceSession(fnModel=proto, dataFlow=dataFlow, userOptions=opts, deviceInfo=device) ir = json.loads(session._serializeIr(popart.IrSerializationFormat.JSON)) for op in ir["maingraph"]: print(op) assert (int(op["attributes"]["__ipu_number"]) == 0)
def reset(self): popart.getLogger().debug("Resetting the dataloaderiterator") # Drain the workers self.flush_data_queue = True while self.batches_outstanding > 0: self.__next__() self.flush_data_queue = False # Reset the sample iterator self.send_idx = 0 self.rcvd_idx = 0 self.sample_iter = iter(self.batch_sampler) # prime the prefetch loop for _ in range(2 * self.num_workers): self._put_indices()
def trainSession(anchors, optimizer, stepSize): popart.getLogger().setLevel("TRACE") builder = popart.Builder() dataShape = popart.TensorInfo("FLOAT", [1, 2, 4, 4]) i1 = builder.addInputTensor(dataShape) filtInit = np.ones([2, 2, 3, 3], dtype=np.float32) i2 = builder.addInitializedInputTensor(filtInit) c1 = builder.aiOnnx.conv([i1, i2], dilations=[1, 1], pads=[1, 1, 1, 1], strides=[1, 1]) c2 = builder.aiOnnx.conv([c1, i2], dilations=[1, 1], pads=[1, 1, 1, 1], strides=[1, 1]) o = builder.aiGraphcore.l1loss([c2], 0.1) proto = builder.getModelProto() opts = popart.SessionOptions() session = popart.TrainingSession( fnModel=proto, dataFlow=popart.DataFlow(stepSize, anchors), loss=o, optimizer=optimizer, deviceInfo=tu.create_test_device(opts={"compileIPUCode": False})) session.prepareDevice() session.weightsFromHost() # add step dimension to infeed infeedShape = dataShape.shape() infeedShape.insert(0, stepSize) data = np.ones(infeedShape, dtype=np.float32) inputs = {i1: data} return session, inputs
def run(torchWriter, patterns, outputdir, cifarInIndices, device, device_hw_id, mode="train", syntheticData=False, transformations=[], epochs=4, printAnchorArrays=False): popart.getLogger().setLevel("TRACE") popart.getLogger("session").setLevel("WARN") if outputdir is None: with TemporaryDirectory() as outputdir: return _run_impl(torchWriter, patterns, outputdir, cifarInIndices, device, device_hw_id, mode, syntheticData, transformations, epochs, printAnchorArrays=printAnchorArrays) else: if not os.path.exists(outputdir): os.mkdir(outputdir) return _run_impl(torchWriter, patterns, outputdir, cifarInIndices, device, device_hw_id, mode, syntheticData, transformations, epochs, printAnchorArrays=printAnchorArrays)
def test_simple_cache_hit(tmp_path, capfd): """ Test: 1. That engine caching works for two identical sessions 2. That the cached engine isn't loaded for a different session """ popart.getLogger().setLevel('DEBUG') opts = popart.SessionOptions() opts.enableEngineCaching = True opts.cachePath = str(tmp_path / 'saved_graph') run_session(2, opts) assert loaded_saved_executable(capfd) is False # Check engine caching works for two identical sessions. run_session(2, opts) assert loaded_saved_executable(capfd) is True # Check the cached engine isn't loaded for a different session. run_session(70, opts) assert loaded_saved_executable(capfd) is False
def test_no_prepare_device(): popart.getLogger().setLevel("TRACE") # Check that `session.modelToHost` can be called when using a # model with a constant node, without throwing an exceptions builder = popart.Builder() i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [2, 2])) i2 = builder.addInputTensor(popart.TensorInfo("FLOAT", [2, 2])) c = builder.aiOnnx.constant(np.array([[1, 2], [3, 4]], dtype=np.float32)) o1 = builder.aiOnnx.add([i1, i2]) o2 = builder.aiOnnx.add([o1, c]) loss = builder.aiGraphcore.identityloss([o2]) proto = builder.getModelProto() anchors = {o2: popart.AnchorReturnType("All")} dataFlow = popart.DataFlow(1, anchors) optimizer = popart.ConstSGD(0.01) opts = popart.SessionOptions() session = popart.TrainingSession(fnModel=proto, dataFlow=dataFlow, userOptions=opts, loss=loss, optimizer=optimizer, deviceInfo=tu.create_test_device()) # No session.prepareDevice() with pytest.raises(popart.popart_exception) as e_info: session.modelToHost('session_proto.onnx') assert (e_info.value.args[0].startswith( "Devicex::prepare() must be called before Devicex::weightsToHost"))
def test_no_virtual_graph(): popart.getLogger().setLevel("TRACE") builder = popart.Builder() i1 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1])) i2 = builder.addInputTensor(popart.TensorInfo("FLOAT", [1])) o1 = builder.aiOnnx.add([i1, i2]) o2 = builder.aiOnnx.add([i1, i2]) o = builder.aiOnnx.add([o1, o2]) builder.addOutputTensor(o) proto = builder.getModelProto() dataFlow = popart.DataFlow(1, {o: popart.AnchorReturnType("All")}) opts = popart.SessionOptions() popart.InferenceSession(fnModel=proto, dataFlow=dataFlow, userOptions=opts, deviceInfo=tu.create_test_device())
def test_verify_synthetic_inputs(capfd, inputType): """ For each synthetic data mode: 1. Get a session that prints the input tensor value to stderr 2. Capture the tensor data from stderr 3. Verify that the data is as expected for that synthetic data mode """ # Hopefully this is large enough to achieve desired tolerance for mean/std, # even for ints. d_shape = [4000] # Test depends on logging output. Silence the logging from PopART popart.getLogger().setLevel("OFF") ## A) Expect input is all zeros run_pt_session(popart.SyntheticDataMode.Zeros, inputType=inputType, d_shape=d_shape) _, err0 = capfd.readouterr() zeroData = numpy_array_from_printtensor_string(err0) assert np.all(zeroData == 0) ## B) Expect input is random normal, T~N(0,1) if inputType == _UINT8: # Casting normal data to unsigned results in non-normal data. return run_pt_session(popart.SyntheticDataMode.RandomNormal, inputType=inputType, d_shape=d_shape) _, err1 = capfd.readouterr() rnData = numpy_array_from_printtensor_string(err1) assert np.all(rnData == 0) == False assert np.isclose(np.mean(rnData), 0, atol=0.02) assert np.isclose(np.std(rnData), 1, atol=0.1)
def __next__(self): if self.num_workers == 0: # same-process loading indices = next(self.sample_iter) # may raise StopIteration if self.log_statisics: t = time.time() batch = self.collate_fn([self.dataset[i] for i in indices]) self.processing_times.append(time.time() - t) if len(self.processing_times) > 8: popart.getLogger().info( "DataLoader processing:{0:6.4f}".format( sum(self.processing_times) / len(self.processing_times))) self.processing_times.clear() else: batch = self.collate_fn([self.dataset[i] for i in indices]) return [x.numpy() for x in batch] # check if the next sample has already been generated if self.rcvd_idx in self.reorder_dict: batch = self.reorder_dict.pop(self.rcvd_idx) return self._process_next_batch(batch) # raise the stop iteration execption when we have recevied all # batches in the data set if self.batches_outstanding == 0: raise StopIteration while True: assert (not self.shutdown and self.batches_outstanding > 0) idx, batch = self._get_batch() self.batches_outstanding -= 1 if idx != self.rcvd_idx: # store out-of-order samples self.reorder_dict[idx] = batch continue return self._process_next_batch(batch)
def test_auto_virtual_graph_train(): ipus = 2 popart.getLogger().setLevel("TRACE") builder = popart.Builder() input_shape = [1, 64] input = builder.addInputTensor(popart.TensorInfo("FLOAT16", input_shape)) x = input for i in range(ipus): w = builder.addInitializedInputTensor(np.zeros([64, 64], np.float16)) x = builder.aiOnnx.matmul([x, w]) output = x builder.addOutputTensor(output) loss = builder.aiGraphcore.identityloss([output]) proto = builder.getModelProto() dataFlow = popart.DataFlow(1, {loss: popart.AnchorReturnType("Final")}) opts = popart.SessionOptions() opts.virtualGraphMode = popart.VirtualGraphMode.Auto device = tu.create_test_device(numIpus=ipus) popart.TrainingSession(fnModel=proto, dataFlow=dataFlow, userOptions=opts, loss=loss, optimizer=popart.SGD( {"defaultLearningRate": (0.01, True)}), deviceInfo=device)
def test_verify_synthetic_inputs(capfd): """ For each synthetic data mode: 1. Get a session that prints the input tensor value to stderr 2. Capture the tensor data from stderr 3. Verify that the data is as expected for that synthetic data mode """ # Test depends on logging output. Silence the logging from PopART popart.getLogger().setLevel("OFF") ## A) Expect input is all zeros run_pt_session(popart.SyntheticDataMode.Zeros) _, err0 = capfd.readouterr() zeroData = numpy_array_from_printtensor_string(err0) assert np.all(zeroData == 0) ## B) Expect input is random normal, T~N(0,1) run_pt_session(popart.SyntheticDataMode.RandomNormal) _, err1 = capfd.readouterr() rnData = numpy_array_from_printtensor_string(err1) assert np.all(rnData == 0) == False assert np.isclose(np.mean(rnData), 0, atol=0.02) assert np.isclose(np.std(rnData), 1, atol=0.1)