def test_mnist_train_pipelined_processes(self): """Generic test on default arguments in training""" cmd = "horovodrun -np 2 -H localhost:2 python horovod_popart_mnist.py --num-ipus 2 --pipeline --epochs 3" output = self.run_command(cmd, self.cwd, "Accuracy") expected_accuracy = [71.26, 78.77, 82.14] accuracies = parse_results_with_regex(output, r".* + Accuracy=+([\d.]+)%") verify_model_accuracies(accuracies[0], expected_accuracy, self.accuracy_tolerances)
def test_mnist_train_sixteen_processes(self): """Generic test on default arguments in training""" cmd = "horovodrun -np 16 -H localhost:16 python horovod_popart_mnist.py --epochs 3" output = self.run_command(cmd, self.cwd, "Accuracy") expected_accuracy = [83.75, 83.86, 84.79] accuracies = parse_results_with_regex(output, r".* + Accuracy=+([\d.]+)%") verify_model_accuracies(accuracies[0], expected_accuracy, self.accuracy_tolerances)
def test_pipelining_convergence(self): """Run with default settings and check it converges""" out = run_pipelining_example({}) # Get the final loss loss_regex = r"loss: ([\d.]+)" result = test_util.parse_results_with_regex(out, loss_regex) # Get the last loss loss = result[0][-1] self.assertGreater(loss, 0.001) self.assertLess(loss, 0.02)
def test_mnist_train_multiple_options(self): """Generic test on default arguments in training""" cmd = "horovodrun -np 4 -H localhost:4 python horovod_popart_mnist.py --epochs 8 --batch-size 64 --batches-per-step 50 --num-ipus 2 --pipeline --log-graph-trace" output = self.run_command(cmd, self.cwd, "Accuracy") expected_accuracy = [ 69.80, 77.77, 81.21, 83.01, 84.11, 84.76, 85.48, 86.02 ] accuracies = parse_results_with_regex(output, r".* + Accuracy=+([\d.]+)%") verify_model_accuracies(accuracies[0], expected_accuracy, self.accuracy_tolerances)