def test_mpi_run_minimal(self):
if not mpi_available():
self.skipTest("MPI is not available")
cmd = ['cmd']
settings = self.minimal_settings
def mpi_impl_flags(tcp, env=None):
return ["--mock-mpi-impl-flags"], ["--mock-mpi-binding-args"]
with mock.patch("horovod.runner.mpi_run._get_mpi_implementation_flags", side_effect=mpi_impl_flags):
with mock.patch("horovod.runner.mpi_run.safe_shell_exec.execute", return_value=0) as execute:
mpi_run(settings, None, {}, cmd)
# call the mocked _get_mpi_implementation_flags method
mpi_flags, binding_args = horovod.runner.mpi_run._get_mpi_implementation_flags(False)
self.assertIsNotNone(mpi_flags)
expected_cmd = ('mpirun '
'--allow-run-as-root --tag-output '
'-np 2 -H localhost:2 '
'{binding_args} '
'{mpi_flags} '
'cmd').format(binding_args=' '.join(binding_args), mpi_flags=' '.join(mpi_flags))
# remove PYTHONPATH from execute's env
# we cannot know the exact value of that env variable
# we test right handling of PYTHONPATH in test_mpi_run_*pythonpath* below
self.assertIn('env', execute.call_args.kwargs)
if 'PYTHONPATH' in execute.call_args.kwargs['env']:
execute.call_args.kwargs['env'].pop('PYTHONPATH')
expected_env = {'PATH': os.environ.get('PATH')}
execute.assert_called_once_with(expected_cmd, env=expected_env, stdout=None, stderr=None)
def test_run_failure(self, controller, mode, run):
if controller == 'gloo' and not gloo_built():
self.skipTest("Gloo is not available")
if controller == 'mpi':
if not (mpi_built() and mpi_available()):
self.skipTest("MPI is not available")
self.do_test_run_with_controller_failure(controller, mode, run)
def test_mpi_run_full(self):
if not mpi_available():
self.skipTest("MPI is not available")
cmd = ['cmd', 'arg1', 'arg2']
nics = ['eth0', 'eth1']
env = {'env1': 'val1', 'env2': 'val2'}
stdout = '<stdout>'
stderr = '<stderr>'
tmout = timeout.Timeout(5, message='Timed out waiting for something.')
settings = hvd_settings.Settings(
verbose=0,
ssh_port=1022,
extra_mpi_args='>mpi-extra args go here<',
binding_args='>binding args go here<',
key=secret.make_secret_key(),
start_timeout=tmout,
num_proc=1,
hosts='localhost:1',
output_filename='>output filename goes here<',
run_func_mode=True
)
def mpi_impl_flags(tcp, env=None):
return ["--mock-mpi-impl-flags"], []
with mock.patch("horovod.runner.mpi_run._get_mpi_implementation_flags", side_effect=mpi_impl_flags) as impl:
with mock.patch("horovod.runner.mpi_run.safe_shell_exec.execute", return_value=0) as execute:
mpi_run(settings, nics, env, cmd, stdout=stdout, stderr=stderr)
# assert call on _get_mpi_implementation_flags
impl.assert_called_once_with(None, env=env)
# call the mocked _get_mpi_implementation_flags method ourselves
mpi_flags, _ = horovod.runner.mpi_run._get_mpi_implementation_flags(False)
self.assertIsNotNone(mpi_flags)
expected_command = ('mpirun '
'--allow-run-as-root --tag-output '
'-np 1 -H {hosts} '
'>binding args go here< '
'{mpi_flags} '
'-mca plm_rsh_args "-p 1022" '
'-mca btl_tcp_if_include eth0,eth1 -x NCCL_SOCKET_IFNAME=eth0,eth1 '
'--output-filename >output filename goes here< '
'-x env1 -x env2 '
'>mpi-extra args go here< '
'cmd arg1 arg2').format(hosts=settings.hosts,
mpi_flags=' '.join(mpi_flags))
# remove PYTHONPATH from execute's env
# we cannot know the exact value of that env variable
# we test right handling of PYTHONPATH in test_mpi_run_*pythonpath* below
self.assertIn('env', execute.call_args.kwargs)
if 'PYTHONPATH' in execute.call_args.kwargs['env']:
execute.call_args.kwargs['env'].pop('PYTHONPATH')
expected_env = {'env1': 'val1', 'env2': 'val2', 'PATH': os.environ.get('PATH')}
execute.assert_called_once_with(expected_command, env=expected_env, stdout=stdout, stderr=stderr)
def test_run_success(self, controller, mode, run):
if controller == 'gloo' and not gloo_built():
self.skipTest("Gloo is not available")
if controller == 'mpi':
if not (mpi_built() and mpi_available()):
self.skipTest("MPI is not available")
if is_mpich():
self.skipTest("MPICH is not testable")
self.do_test_run_with_controller_success(controller, mode, run)
def test_mpi_run_with_os_environ(self):
if not mpi_available():
self.skipTest("MPI is not available")
cmd = ['cmd']
settings = self.minimal_settings
def mpi_impl_flags(tcp, env=None):
return ["--mock-mpi-impl-flags"], ["--mock-mpi-binding-args"]
with mock.patch("horovod.runner.mpi_run._get_mpi_implementation_flags", side_effect=mpi_impl_flags):
with mock.patch("horovod.runner.mpi_run.safe_shell_exec.execute", return_value=0):
with pytest.raises(Exception, match="^env argument must be a dict, not <class 'os._Environ'>: "):
mpi_run(settings, None, os.environ, cmd)
def test_mpi_run_with_non_zero_exit(self):
if not mpi_available():
self.skipTest("MPI is not available")
cmd = ['cmd']
settings = self.minimal_settings
def mpi_impl_flags(tcp, env=None):
return [], []
with mock.patch("horovod.runner.mpi_run._get_mpi_implementation_flags", side_effect=mpi_impl_flags):
with mock.patch("horovod.runner.mpi_run.safe_shell_exec.execute", return_value=1):
with pytest.raises(RuntimeError, match="^mpirun failed with exit code 1$"):
mpi_run(settings, None, {}, cmd)
def test_run_failure(self, controller, mode, run):
if controller == 'gloo' and not gloo_built():
self.skipTest("Gloo is not available")
if controller == 'mpi':
if not (mpi_built() and mpi_available()):
self.skipTest("MPI is not available")
if is_mpich():
self.skipTest("MPICH is not testable")
if is_intel_mpi():
self.skipTest(
"Intel(R) MPI is not testable because it is based on MPICH"
)
self.do_test_run_with_controller_failure(controller, mode, run)
def do_test_mpi_run_env_override(self, sysenv, argenv, env_var, expected):
if not mpi_available():
self.skipTest("MPI is not available")
cmd = ['cmd']
settings = self.minimal_settings
def mpi_impl_flags(tcp, env=None):
return ["--mock-mpi-impl-flags"], ["--mock-mpi-binding-args"]
with mock.patch("horovod.runner.mpi_run._get_mpi_implementation_flags", side_effect=mpi_impl_flags),\
mock.patch("horovod.runner.mpi_run.safe_shell_exec.execute", return_value=0) as execute,\
override_env(sysenv):
mpi_run(settings, None, argenv, cmd)
# assert the env variable in the execute's env
self.assertIn('env', execute.call_args.kwargs)
self.assertEqual(execute.call_args.kwargs['env'].get(env_var), expected)
