def init(self,
rendezvous_addr=None,
rendezvous_port=None,
nic=None,
hostname=None,
local_rank=None):
with self._lock:
if self._service:
return
rendezvous_addr = rendezvous_addr or os.environ.get(
HOROVOD_GLOO_RENDEZVOUS_ADDR)
if not rendezvous_addr:
return
rendezvous_port = rendezvous_port if rendezvous_port is not None else \
int(os.environ.get(HOROVOD_GLOO_RENDEZVOUS_PORT))
nic = nic or os.environ.get(HOROVOD_GLOO_IFACE)
hostname = hostname or os.environ.get(HOROVOD_HOSTNAME)
local_rank = local_rank if local_rank is not None else \
int(os.environ.get(HOROVOD_LOCAL_RANK))
secret_key = secret.make_secret_key()
self._service = WorkerNotificationService(secret_key, nic, self)
value = (self._service.addresses(), secret_key)
put_data_into_kvstore(rendezvous_addr, rendezvous_port,
PUT_WORKER_ADDRESSES,
self._create_id(hostname, local_rank), value)
def main(addr, port):
func = read_data_from_kvstore(addr, port, 'runfunc', 'func')
try:
ret_val = func()
except BaseException as e:
sys.stderr.write("User function raise error: {error}".format(error=str(e)))
raise e
rank, size = get_env_rank_and_size()
put_data_into_kvstore(addr, port, 'runfunc_result', str(rank), ret_val)
def _run(args):
if args.check_build:
check_build(args.verbose)
# if hosts are not specified, either parse from hostfile, or default as
# localhost
if not args.hosts:
if args.hostfile:
args.hosts = parse_host_files(args.hostfile)
else:
# Set hosts to localhost if not specified
args.hosts = 'localhost:{np}'.format(np=args.np)
host_list = args.hosts.split(',')
all_host_names = []
pattern = re.compile(r'^[\w.-]+:\d+$')
for host in host_list:
if not pattern.match(host.strip()):
raise ValueError('Invalid host input, please make sure it has '
'format as : worker-0:2,worker-1:2.')
all_host_names.append(host.strip().split(':')[0])
# horovodrun has to finish all the checks before this timeout runs out.
if args.start_timeout:
start_timeout = args.start_timeout
else:
# Lookup default timeout from the environment variable.
start_timeout = int(os.getenv('HOROVOD_START_TIMEOUT', '30'))
tmout = timeout.Timeout(start_timeout,
message='Timed out waiting for {activity}. Please '
'check connectivity between servers. You '
'may need to increase the --start-timeout '
'parameter if you have too many servers.')
settings = hvd_settings.Settings(verbose=2 if args.verbose else 0,
ssh_port=args.ssh_port,
ssh_ports=args.ssh_ports,
extra_mpi_args=args.mpi_args,
key=secret.make_secret_key(),
timeout=tmout,
num_hosts=len(all_host_names),
num_proc=args.np,
hosts=args.hosts,
output_filename=args.output_filename,
run_func_mode=args.run_func is not None,
nic=args.nic)
# This cache stores the results of checks performed by horovodrun
# during the initialization step. It can be disabled by setting
# --disable-cache flag.
fn_cache = None
if not args.disable_cache:
params = ''
if args.np:
params += str(args.np) + ' '
if args.hosts:
params += str(args.hosts) + ' '
if args.ssh_port:
params += str(args.ssh_port)
elif args.ssh_ports:
params += str(args.ssh_ports)
parameters_hash = hashlib.md5(params.encode('utf-8')).hexdigest()
fn_cache = cache.Cache(CACHE_FOLDER, CACHE_STALENESS_THRESHOLD_MINUTES,
parameters_hash)
if settings.verbose >= 2:
print('Filtering local host names.')
remote_host_names = network.filter_local_addresses(all_host_names)
if settings.verbose >= 2:
print('Remote host found: ' + ' '.join(remote_host_names))
if len(remote_host_names) > 0:
if settings.verbose >= 2:
print('Checking ssh on all remote hosts.')
# Check if we can ssh into all remote hosts successfully.
if args.ssh_ports:
ssh_ports = [
port for host, port in zip(all_host_names,
args.ssh_ports.split(","))
if host in set(remote_host_names)
]
ssh_ports = ",".join(ssh_ports)
else:
ssh_ports = None
_check_all_hosts_ssh_successful(remote_host_names,
ssh_port=args.ssh_port,
ssh_ports=ssh_ports,
fn_cache=fn_cache)
if settings.verbose >= 2:
print('SSH was successful into all the remote hosts.')
if len(remote_host_names) > 0:
if settings.verbose >= 2:
print('Testing interfaces on all the hosts.')
local_host_names = set(all_host_names) - set(remote_host_names)
# Find the set of common, routed interfaces on all the hosts (remote
# and local) and specify it in the args to be used by NCCL. It is
# expected that the following function will find at least one interface
# otherwise, it will raise an exception.
common_intfs = _driver_fn(all_host_names,
local_host_names,
settings,
fn_cache=fn_cache)
if settings.verbose >= 2:
print('Interfaces on all the hosts were successfully checked.')
print('Common interface found: ' + ' '.join(common_intfs))
else:
if settings.verbose >= 2:
print('All hosts are local, finding the interfaces '
'with address 127.0.0.1')
# If all the given hosts are local, find the interfaces with address
# 127.0.0.1
common_intfs = set()
for iface, addrs in net_if_addrs().items():
if settings.nic and iface != settings.nic:
continue
for addr in addrs:
if addr.family == AF_INET and addr.address == '127.0.0.1':
common_intfs.add(iface)
break
if len(common_intfs) == 0:
raise ValueError('No interface is found for address 127.0.0.1.')
if settings.verbose >= 2:
print('Local interface found ' + ' '.join(common_intfs))
# get the driver IPv4 address
driver_ip = _get_driver_ip(common_intfs)
if args.run_func:
run_func_server = KVStoreServer(verbose=settings.verbose)
run_func_server_port = run_func_server.start_server()
pickled_exec_func = cloudpickle.dumps(args.run_func)
put_data_into_kvstore(driver_ip, run_func_server_port, 'runfunc',
'func', pickled_exec_func)
command = [
sys.executable, '-m', 'horovod.run.run_task',
str(driver_ip),
str(run_func_server_port)
]
try:
_launch_job(args, remote_host_names, settings, common_intfs,
command)
results = [None] * args.np
# TODO: make it parallel to improve performance
for i in range(args.np):
pickled_result = read_data_from_kvstore(
driver_ip, run_func_server_port, 'runfunc_result', str(i))
results[i] = cloudpickle.loads(pickled_result)
return results
finally:
run_func_server.shutdown_server()
else:
command = args.command
_launch_job(args, remote_host_names, settings, common_intfs, command)
return None
def _run_static(args):
all_host_names, _ = parse_hosts_and_slots(args.hosts)
nics_set = set(args.nics.split(',')) if args.nics else None
# horovodrun has to finish all the checks before this timeout runs out.
if args.start_timeout:
start_timeout = args.start_timeout
else:
# Lookup default timeout from the environment variable.
start_timeout = int(os.getenv('HOROVOD_START_TIMEOUT', '30'))
tmout = timeout.Timeout(start_timeout,
message='Timed out waiting for {activity}. Please '
'check connectivity between servers. You '
'may need to increase the --start-timeout '
'parameter if you have too many servers.')
settings = hvd_settings.Settings(verbose=2 if args.verbose else 0,
ssh_port=args.ssh_port,
extra_mpi_args=args.mpi_args,
tcp_flag=args.tcp_flag,
binding_args=args.binding_args,
key=secret.make_secret_key(),
start_timeout=tmout,
num_proc=args.np,
hosts=args.hosts,
num_hosts=len(all_host_names),
output_filename=args.output_filename,
run_func_mode=args.run_func is not None,
nics=nics_set)
# This cache stores the results of checks performed by horovod
# during the initialization step. It can be disabled by setting
# --disable-cache flag.
fn_cache = None
if not args.disable_cache:
params = ''
if args.np:
params += str(args.np) + ' '
if args.hosts:
params += str(args.hosts) + ' '
if args.ssh_port:
params += str(args.ssh_port)
parameters_hash = hashlib.md5(params.encode('utf-8')).hexdigest()
fn_cache = cache.Cache(CACHE_FOLDER, CACHE_STALENESS_THRESHOLD_MINUTES,
parameters_hash)
if settings.verbose >= 2:
print('Filtering local host names.')
remote_host_names = network.filter_local_addresses(all_host_names)
if settings.verbose >= 2:
print('Remote host found: ' + ' '.join(remote_host_names))
if len(remote_host_names) > 0:
if settings.verbose >= 2:
print('Checking ssh on all remote hosts.')
# Check if we can ssh into all remote hosts successfully.
_check_all_hosts_ssh_successful(remote_host_names,
args.ssh_port,
fn_cache=fn_cache)
if settings.verbose >= 2:
print('SSH was successful into all the remote hosts.')
nics = driver_service.get_common_interfaces(settings, all_host_names,
remote_host_names, fn_cache)
if args.run_func:
# get the driver IPv4 address
driver_ip = network.get_driver_ip(nics)
run_func_server = KVStoreServer(verbose=settings.verbose)
run_func_server_port = run_func_server.start_server()
put_data_into_kvstore(driver_ip, run_func_server_port, 'runfunc',
'func', args.run_func)
command = [
sys.executable, '-m', 'horovod.run.run_task',
str(driver_ip),
str(run_func_server_port)
]
try:
_launch_job(args, settings, nics, command)
results = [None] * args.np
# TODO: make it parallel to improve performance
for i in range(args.np):
results[i] = read_data_from_kvstore(driver_ip,
run_func_server_port,
'runfunc_result', str(i))
return results
finally:
run_func_server.shutdown_server()
else:
command = args.command
_launch_job(args, settings, nics, command)
return None
def _run(args):
if args.check_build:
check_build(args.verbose)
# If LSF is used, use default values from job config
if lsf.LSFUtils.using_lsf():
if not args.np:
args.np = lsf.LSFUtils.get_num_processes()
if not args.hosts and not args.hostfile:
args.hosts = ','.join(
'{host}:{np}'.format(host=host, np=lsf.LSFUtils.get_num_gpus())
for host in lsf.LSFUtils.get_compute_hosts())
# if hosts are not specified, either parse from hostfile, or default as
# localhost
if not args.hosts:
if args.hostfile:
args.hosts = parse_host_files(args.hostfile)
else:
# Set hosts to localhost if not specified
args.hosts = 'localhost:{np}'.format(np=args.np)
host_list = args.hosts.split(',')
all_host_names = []
pattern = re.compile(r'^[\w.-]+:\d+$')
for host in host_list:
if not pattern.match(host.strip()):
raise ValueError('Invalid host input, please make sure it has '
'format as : worker-0:2,worker-1:2.')
all_host_names.append(host.strip().split(':')[0])
# horovodrun has to finish all the checks before this timeout runs out.
if args.start_timeout:
start_timeout = args.start_timeout
else:
# Lookup default timeout from the environment variable.
start_timeout = int(os.getenv('HOROVOD_START_TIMEOUT', '30'))
tmout = timeout.Timeout(start_timeout,
message='Timed out waiting for {activity}. Please '
'check connectivity between servers. You '
'may need to increase the --start-timeout '
'parameter if you have too many servers.')
settings = hvd_settings.Settings(verbose=2 if args.verbose else 0,
ssh_port=args.ssh_port,
extra_mpi_args=args.mpi_args,
tcp_flag=args.tcp_flag,
binding_args=args.binding_args,
key=secret.make_secret_key(),
timeout=tmout,
num_hosts=len(all_host_names),
num_proc=args.np,
hosts=args.hosts,
output_filename=args.output_filename,
run_func_mode=args.run_func is not None,
nic=args.nic)
# This cache stores the results of checks performed by horovodrun
# during the initialization step. It can be disabled by setting
# --disable-cache flag.
fn_cache = None
if not args.disable_cache:
params = ''
if args.np:
params += str(args.np) + ' '
if args.hosts:
params += str(args.hosts) + ' '
if args.ssh_port:
params += str(args.ssh_port)
parameters_hash = hashlib.md5(params.encode('utf-8')).hexdigest()
fn_cache = cache.Cache(CACHE_FOLDER, CACHE_STALENESS_THRESHOLD_MINUTES,
parameters_hash)
if settings.verbose >= 2:
print('Filtering local host names.')
remote_host_names = network.filter_local_addresses(all_host_names)
if settings.verbose >= 2:
print('Remote host found: ' + ' '.join(remote_host_names))
if len(remote_host_names) > 0:
if settings.verbose >= 2:
print('Checking ssh on all remote hosts.')
# Check if we can ssh into all remote hosts successfully.
_check_all_hosts_ssh_successful(remote_host_names,
args.ssh_port,
fn_cache=fn_cache)
if settings.verbose >= 2:
print('SSH was successful into all the remote hosts.')
common_intfs = driver_service._get_common_interfaces(
settings, all_host_names, remote_host_names, fn_cache)
if args.run_func:
# get the driver IPv4 address
driver_ip = network._get_driver_ip(common_intfs)
run_func_server = KVStoreServer(verbose=settings.verbose)
run_func_server_port = run_func_server.start_server()
pickled_exec_func = cloudpickle.dumps(args.run_func)
put_data_into_kvstore(driver_ip, run_func_server_port, 'runfunc',
'func', pickled_exec_func)
command = [
sys.executable, '-m', 'horovod.run.run_task',
str(driver_ip),
str(run_func_server_port)
]
try:
_launch_job(args, remote_host_names, settings, common_intfs,
command)
results = [None] * args.np
# TODO: make it parallel to improve performance
for i in range(args.np):
pickled_result = read_data_from_kvstore(
driver_ip, run_func_server_port, 'runfunc_result', str(i))
results[i] = cloudpickle.loads(pickled_result)
return results
finally:
run_func_server.shutdown_server()
else:
command = args.command
_launch_job(args, remote_host_names, settings, common_intfs, command)
return None
