def test_pipelining_convergence(self): """Run with default settings and check it converges""" out = run_pipelining_example({}) # Get the final loss loss_regex = r"loss: ([\d.]+)" result = test_util.parse_results_with_regex(out, loss_regex) # Get the last loss loss = result[0][-1] assert 0.001 <= loss <= 0.020
def test_mnist_train_pipelined_processes(self): """Generic test on default arguments in training""" cmd = "horovodrun -np 2 -H localhost:2 python horovod_popart_mnist.py --num-ipus 2 --pipeline --epochs 3" output = self.run_command(cmd, self.cwd, "Accuracy") expected_accuracy = [71.26, 78.77, 82.14] accuracies = parse_results_with_regex(output, r".* + Accuracy=+([\d.]+)%") verify_model_accuracies(accuracies[0], expected_accuracy, self.accuracy_tolerances)
def test_mnist_train_sixteen_processes(self): """Generic test on default arguments in training""" cmd = "horovodrun -np 16 -H localhost:16 python horovod_popart_mnist.py --epochs 3" output = self.run_command(cmd, self.cwd, "Accuracy") expected_accuracy = [83.75, 83.86, 84.79] accuracies = parse_results_with_regex(output, r".* + Accuracy=+([\d.]+)%") verify_model_accuracies(accuracies[0], expected_accuracy, self.accuracy_tolerances)
def test_mnist_train_multiple_options(self): """Generic test on default arguments in training""" cmd = "horovodrun -np 4 -H localhost:4 python horovod_popart_mnist.py --epochs 8 --batch-size 64 --batches-per-step 50 --num-ipus 2 --pipeline --log-graph-trace" output = self.run_command(cmd, self.cwd, "Accuracy") expected_accuracy = [ 69.80, 77.77, 81.21, 83.01, 84.11, 84.76, 85.48, 86.02 ] accuracies = parse_results_with_regex(output, r".* + Accuracy=+([\d.]+)%") verify_model_accuracies(accuracies[0], expected_accuracy, self.accuracy_tolerances)
def get_results(output): """Finds the average loss, items/sec and tokens/sec in the output and returns the final average loss and the average items/sec and tokens/sec respectively""" line_regex = re.compile( r"Step:\s*\d+. Average Loss ([\d.]+). " ) results = parse_results_with_regex(output, line_regex) avg_loss_list = results[0] return avg_loss_list[-1]
def get_results(output): """Finds the average loss, items/sec and tokens/sec in the output and returns the final average loss and the average items/sec and tokens/sec respectively""" line_regex = re.compile(r"Step: \d+. Average Loss ([\d.]+). " r"Items/sec ([\d.]+). Tokens/sec ([\d.]+)") results = test_util.parse_results_with_regex(output, line_regex) avg_loss_list = results[0] items_sec_list = results[1] tokens_sec_list = results[2] return avg_loss_list[-1], mean(items_sec_list), mean(tokens_sec_list)