def test_mnist_all_data(self):
"""Generic test using all the available data (10,000)"""
py_args = self.generic_arguments.copy()
py_args["--epochs"] = 2
py_args["--batch-size"] = 10
py_args["--batches-per-step"] = 1000
test_util.run_test_helper(run_popart_mnist_training, **py_args)
def _arg_test_helper(self, *args):
"""Helper function that exercises the command line arguments in the
python model"""
py_args = self.generic_arguments.copy()
for arg in args:
py_args[arg] = ""
run_test_helper(run_tensorflow_nmt, **py_args)
def test_mnist_conv_simulation(self):
"""Simulation test with basic arguments - This simulation takes
around 838s (~14 minutes) complete"""
py_args = self.generic_arguments.copy()
py_args["--simulation"] = ""
py_args["--batch-size"] = 1
py_args["--batches-per-step"] = 1
py_args["--epochs"] = 1
test_util.run_test_helper(run_popart_mnist_training, **py_args)
def test_train_32_100_1_simulation(self):
"""Generic test in simulation mode"""
py_args = self.generic_arguments.copy()
py_args["--simulation"] = ""
py_args["--epochs"] = 1
run_test_helper(
run_pytorch_mnist,
**py_args
)
def test_log_graph_trace_arg(self):
"""Generic test exercising log graph trace argument"""
py_args = self.generic_arguments.copy()
py_args["--log-graph-trace"] = ""
py_args["--epochs"] = 1
run_test_helper(
run_pytorch_mnist,
**py_args
)
def train_and_infer_helper(
py_args,
last_measured_train_time=None,
last_measured_infer_time=None,
last_measured_loss=None,
time_tolerances=0.6,
infer=True,
):
"""Helper function for running training followed by
inference tests
Runs the NMT Tensorflow model with arguments py_arg. The time taken
and output are processed and verified against their respective previous
values.
Args:
py_args: dictionary with string keys and values. Each item
represents an argument to be used when the model is run
last_measured_train_time: float representing the previously
measured time to train the model
last_measured_infer_time: float representing the previously
measured time to perform inference on the model
last_measured_loss: float representing the previously measured
final loss of the model in training
time_tolerances: float representing the percentage tolerance
on the previously measured values to be asserted against
infer: bool perform inference if True
Returns:
None
Raises:
AssertionError: If the measured values are out of the range of the
previously measured values with the tolerance applied
"""
out = run_test_helper(
run_tensorflow_nmt, total_run_time=last_measured_train_time, **py_args
)
average_loss = get_results(out)
if last_measured_loss:
loss_minimum = get_minimum_with_tolerance(last_measured_loss, 0.2)
loss_maximum = get_maximum_with_tolerance(last_measured_loss, 0.2)
assert average_loss >= loss_minimum
assert average_loss <= loss_maximum
if infer:
py_args["--infer"] = ""
run_test_helper(
run_tensorflow_nmt, total_run_time=last_measured_infer_time, **py_args
)
def test_train_32_100_10(self):
"""Generic functional test"""
py_args = self.generic_arguments.copy()
out = test_util.run_test_helper(run_pytorch_mnist, **py_args)
expected_accuracy = [
87.65, 88.06, 88.4, 88.43, 88.68, 88.71, 88.69, 88.89, 88.85, 88.61
]
test_util.parse_results_for_accuracy(out, expected_accuracy, 2.5)
def test_mnist_train(self):
"""Generic test on default arguments in training"""
py_args = self.generic_arguments.copy()
out = test_util.run_test_helper(run_popart_mnist_training, **py_args)
expected_accuracy = [
97.72, 98.15, 98.51, 98.55, 98.55, 98.38, 98.34, 98.35, 98.43,
98.41
]
test_util.parse_results_for_accuracy(out, expected_accuracy,
self.accuracy_tolerances)
def test_mnist_train(self):
"""Generic test on default arguments in training"""
py_args = self.generic_arguments.copy()
out = run_test_helper(run_popart_mnist_training, **py_args)
expected_accuracy = [
88.88, 89.63, 89.83, 90.01, 90.12, 90.22, 90.40, 90.59, 90.65,
90.70
]
parse_results_for_accuracy(out, expected_accuracy,
self.accuracy_tolerances)
def test_train_torchvision(self):
"""Generic functional test"""
py_args = self.generic_arguments.copy()
py_args["--num-ipus"] = 2
py_args["--epochs"] = 10
out = run_test_helper(run_pytorch_Torchvision, **py_args)
expected_accuracy = [
16.85, 24.52, 30.77, 36.34, 40.65, 45.59, 49.07, 52.13, 55.52,
58.14
]
parse_results_for_accuracy(out, expected_accuracy, 2.5)
def test_mnist_train_sharded_pipelined(self):
"""Generic test on default arguments in training over 2 IPUs
and pipelined"""
py_args = self.generic_arguments.copy()
py_args["--num-ipus"] = 2
py_args["--pipeline"] = ""
out = run_test_helper(run_popart_mnist_training, **py_args)
expected_accuracy = [
88.11, 88.69, 88.91, 88.94, 88.92, 88.98, 89.05, 89.14, 89.18,
89.25
]
parse_results_for_accuracy(out, expected_accuracy,
self.accuracy_tolerances)
def test_mnist_log_graph_trace(self):
"""Basic test with log-graph-trace argument"""
py_args = self.generic_arguments.copy()
py_args["--log-graph-trace"] = ""
py_args["--epochs"] = 1
test_util.run_test_helper(run_popart_mnist_training, **py_args)
def test_mnist_simulation(self):
"""Simulation test with basic arguments"""
py_args = self.generic_arguments.copy()
py_args["--simulation"] = ""
py_args["--epochs"] = 2
run_test_helper(run_popart_mnist_training, **py_args)
