def test_pipelining_convergence(self):
"""Run with default settings and check it converges"""
out = run_pipelining_example({})
# Get the final loss
loss_regex = r"loss: ([\d.]+)"
result = test_util.parse_results_with_regex(out, loss_regex)
# Get the last loss
loss = result[0][-1]
assert 0.001 <= loss <= 0.020
def test_mnist_train_pipelined_processes(self):
"""Generic test on default arguments in training"""
cmd = "horovodrun -np 2 -H localhost:2 python horovod_popart_mnist.py --num-ipus 2 --pipeline --epochs 3"
output = self.run_command(cmd, self.cwd, "Accuracy")
expected_accuracy = [71.26, 78.77, 82.14]
accuracies = parse_results_with_regex(output,
r".* + Accuracy=+([\d.]+)%")
verify_model_accuracies(accuracies[0], expected_accuracy,
self.accuracy_tolerances)
def test_mnist_train_sixteen_processes(self):
"""Generic test on default arguments in training"""
cmd = "horovodrun -np 16 -H localhost:16 python horovod_popart_mnist.py --epochs 3"
output = self.run_command(cmd, self.cwd, "Accuracy")
expected_accuracy = [83.75, 83.86, 84.79]
accuracies = parse_results_with_regex(output,
r".* + Accuracy=+([\d.]+)%")
verify_model_accuracies(accuracies[0], expected_accuracy,
self.accuracy_tolerances)
def test_mnist_train_multiple_options(self):
"""Generic test on default arguments in training"""
cmd = "horovodrun -np 4 -H localhost:4 python horovod_popart_mnist.py --epochs 8 --batch-size 64 --batches-per-step 50 --num-ipus 2 --pipeline --log-graph-trace"
output = self.run_command(cmd, self.cwd, "Accuracy")
expected_accuracy = [
69.80, 77.77, 81.21, 83.01, 84.11, 84.76, 85.48, 86.02
]
accuracies = parse_results_with_regex(output,
r".* + Accuracy=+([\d.]+)%")
verify_model_accuracies(accuracies[0], expected_accuracy,
self.accuracy_tolerances)
def get_results(output):
"""Finds the average loss, items/sec and tokens/sec in the output and
returns the final average loss and the average items/sec and
tokens/sec respectively"""
line_regex = re.compile(
r"Step:\s*\d+. Average Loss ([\d.]+). "
)
results = parse_results_with_regex(output, line_regex)
avg_loss_list = results[0]
return avg_loss_list[-1]
def get_results(output):
"""Finds the average loss, items/sec and tokens/sec in the output and
returns the final average loss and the average items/sec and
tokens/sec respectively"""
line_regex = re.compile(r"Step: \d+. Average Loss ([\d.]+). "
r"Items/sec ([\d.]+). Tokens/sec ([\d.]+)")
results = test_util.parse_results_with_regex(output, line_regex)
avg_loss_list = results[0]
items_sec_list = results[1]
tokens_sec_list = results[2]
return avg_loss_list[-1], mean(items_sec_list), mean(tokens_sec_list)