def test_mnist_train(self): """Generic test on default arguments in training""" py_args = self.generic_arguments.copy() out = test_util.run_test_helper(run_popart_mnist_training, **py_args) expected_accuracy = [98.41] test_util.parse_results_for_accuracy(out, expected_accuracy, self.accuracy_tolerances)
def test_train_32_100_10(self): """Generic functional test""" py_args = self.generic_arguments.copy() out = run_test_helper(run_pytorch_mnist, **py_args) expected_accuracy = [ 87.65, 88.06, 88.4, 88.43, 88.68, 88.71, 88.69, 88.89, 88.85, 88.61 ] parse_results_for_accuracy(out, expected_accuracy, 2.5)
def test_mnist_train(self): """Generic test on default arguments in training""" py_args = self.generic_arguments.copy() out = run_test_helper(run_mnist_training, **py_args) expected_accuracy = [ 50.32, 58.67, 68.09, 72.81, 75.30, 77.14, 78.53, 79.75, 80.62, 81.37 ] parse_results_for_accuracy(out, expected_accuracy, self.accuracy_tolerances)
def test_mnist_train(self): """Generic test on default arguments in training""" py_args = self.generic_arguments.copy() out = run_test_helper(run_popart_mnist_training, **py_args) expected_accuracy = [ 88.88, 89.63, 89.83, 90.01, 90.12, 90.22, 90.40, 90.59, 90.65, 90.70 ] parse_results_for_accuracy(out, expected_accuracy, self.accuracy_tolerances)
def test_mnist_train_sharded(self): """Generic test on default arguments in training over 2 IPUs""" py_args = self.generic_arguments.copy() py_args["--num-ipus"] = 2 out = run_test_helper( run_popart_mnist_training, **py_args ) expected_accuracy = [90.70] parse_results_for_accuracy( out, expected_accuracy, self.accuracy_tolerances )
def test_mnist_train_sharded_pipelined(self): """Generic test on default arguments in training over 2 IPUs and pipelined""" py_args = self.generic_arguments.copy() py_args["--num-ipus"] = 2 py_args["--pipeline"] = "" out = run_test_helper(run_popart_mnist_training, **py_args) expected_accuracy = [ 88.11, 88.69, 88.91, 88.94, 88.92, 88.98, 89.05, 89.14, 89.18, 89.25 ] parse_results_for_accuracy(out, expected_accuracy, self.accuracy_tolerances)
def test_mnist_train_replicated(self): """Generic test on default arguments in training over 2 IPUs with replication""" py_args = self.generic_arguments.copy() py_args["--num-ipus"] = 2 py_args["--replication-factor"] = 2 out = run_test_helper(run_popart_mnist_training, **py_args) expected_accuracy = [ 88.88, 89.63, 89.83, 90.01, 90.12, 90.22, 90.40, 90.59, 90.65, 90.70 ] parse_results_for_accuracy(out, expected_accuracy, self.accuracy_tolerances)
def test_mnist_complete(self): """Tests that the model trains and a pvti file is created""" py_args = self.generic_arguments.copy() out = run_test_helper( run_popart_mnist_complete, **py_args ) expected_accuracy = [88.88] parse_results_for_accuracy( out, expected_accuracy, self.accuracy_tolerances ) # Verify that the pvti file is created if not any(fname.endswith(".pvti") for fname in os.listdir(os.path.dirname(__file__))): raise AssertionError("No pvti file found")
def test_mnist_train_replicated_pipelined(self): """Generic test on default arguments in training over 2 IPUs and pipelined""" py_args = self.generic_arguments.copy() py_args["--num-ipus"] = 4 py_args["--replication-factor"] = 2 py_args["--pipeline"] = "" out = run_test_helper( run_popart_mnist_training, **py_args ) expected_accuracy = [89.25] parse_results_for_accuracy( out, expected_accuracy, self.accuracy_tolerances )
def test_resnet8_bs4_4ipus(self): with TemporaryDirectory() as datadir: out = run_popart_resnet_training( **{ '--size': '8', '--batch-size': 4, '--norm-type': 'GROUP', '--num-ipus': 4, '--epochs': 5, '--no-prng': '', '--data-dir': datadir, '--num-workers': 0 }) expected_accuracy = [48.5, 53.6, 61.6, 64.6, 65.1] parse_results_for_accuracy(out, expected_accuracy, 4)
def test_resnet8_bs4_4ipus_pipeline(self): with TemporaryDirectory() as datadir: out = run_popart_resnet_training( **{ '--size': '8', '--batch-size': 4, '--norm-type': 'GROUP', '--num-ipus': 4, '--epochs': 5, '--pipeline': '', '--no-prng': '', '--data-dir': datadir, '--num-workers': 0, '--ckpt': os.path.join(datadir, "ckpt.onnx"), }) expected_accuracy = [42.6, 48.8, 63.7, 62.5, 61.0] parse_results_for_accuracy(out, expected_accuracy, 6)
def test_resnet8_bs4_4ipus_pipeline_replication(self): with TemporaryDirectory() as datadir: out = run_popart_resnet_training( **{ '--size': '8', '--batch-size': 8, '--norm-type': 'GROUP', '--num-ipus': 8, '--epochs': 5, '--pipeline': '', '--replication-factor': 2, '--no-prng': '', '--data-dir': datadir, '--num-workers': 0 }) expected_accuracy = [42.25, 48.55, 59.4, 62.35, 61.0] parse_results_for_accuracy(out, expected_accuracy, 4)
def run_cifar10(file_name): cwd = os.path.dirname(os.path.abspath(__file__)) cmd = ["python3", 'cifar10_{0}.py'.format(file_name)] out = subprocess.check_output(cmd, cwd=cwd).decode("utf-8") parse_results_for_accuracy(out, [81.0], 6.0) return out