def test_cifar_resnet_distributed(device_id):
params = [ "-datadir", base_path]
str_out = mpiexec_execute(script_under_test, mpiexec_params, params)
#Training loss of the generator at worker: {0} is: {2.201804}, time taken is: {40} seconds
results = re.findall("Training loss of the generator at worker: \{.+?\} is: \{.+?\}", str_out)
assert(len(results) == 4)
def test_finalize_with_exception_no_hang():
str_out = mpiexec_execute(__file__, ["-n", "2"], [])
results = re.findall("Completed with exception.", str_out)
assert len(results) == 1
results = re.findall("Completed successfully.", str_out)
assert len(results) == 0
def test_finalize_with_exception_no_hang():
str_out = mpiexec_execute(__file__, ["-n", "2"], [])
results = re.findall("Completed with exception.", str_out)
assert len(results) == 1
results = re.findall("Completed successfully.", str_out)
assert len(results) == 0
def test_sample_count_with_several_distributed_learners():
str_out = mpiexec_execute(__file__, ["-n", "2"], [])
results = re.findall("Completed with exception.", str_out)
if len(results) != 0:
print(str_out)
assert False
results = re.findall("Completed successfully.", str_out)
if len(results) != 2:
print(str_out)
assert False
def test_sample_count_with_several_distributed_learners():
str_out = mpiexec_execute(__file__, ["-n", "2"], [])
results = re.findall("Completed with exception.", str_out)
if len(results) != 0:
print(str_out)
assert False
results = re.findall("Completed successfully.", str_out)
if len(results) != 2:
print(str_out)
assert False
def test_htk_lstm_truncated_distributed_gpu_with_cv(device_id):
# Make sure that full sequence cross validation
# works in the middle of bptt training
params = [ "-n", "2",
"-datadir", an4_dataset_directory(),
"-q", "1",
"-m", "640",
"-e", "1500",
"-cvfreq", "1000",
"-device", str(device_id) ]
output = mpiexec_execute(device_id=device_id, script=script_under_test, mpiexec_params=mpiexec_params, params=params)
results = re.findall(r"Finished Evaluation \[.+?\]: Minibatch\[.+?\]: metric = (.+?)%", output)
assert len(results) == 6, output
def test_cifar_convnet_distributed_block_momentum(device_id):
params = [ "-n", "1",
"-m", "64",
"-e", "13000",
"-datadir", base_path,
"-b", "1600",
"-r",
"-device", str(device_id) ]
# 13000 samples / 2 worker / 64 mb_size = 101 minibatchs.
# We expect to see only Minibatch[ 1 -100]
output = mpiexec_execute(script_under_test, mpiexec_params, params, device_id=device_id)
results = re.findall(r"Minibatch\[(.+?)\]: loss = .+?%", output)
assert len(results) == 2
assert results[0] == ' 1- 100'
assert results[1] == ' 1- 100'
def test_cifar_convnet_distributed_block_momentum(device_id):
params = [
"-n", "1", "-m", "64", "-e", "13000", "-datadir", base_path, "-b",
"1600", "-r", "-device",
str(device_id)
]
# 13000 samples / 2 worker / 64 mb_size = 101 minibatchs.
# We expect to see only Minibatch[ 1 -100]
output = mpiexec_execute(script_under_test,
mpiexec_params,
params,
device_id=device_id)
results = re.findall("Minibatch\[(.+?)\]: loss = .+?%", output)
assert len(results) == 2
assert results[0] == ' 1- 100'
assert results[1] == ' 1- 100'