def install(self):
Print.info('Installing rust and cloning the repo...')
cmd = [
'sudo apt-get update',
'sudo apt-get -y upgrade',
'sudo apt-get -y autoremove',
# The following dependencies prevent the error: [error: linker `cc` not found].
'sudo apt-get -y install build-essential',
'sudo apt-get -y install cmake',
# Install rust (non-interactive).
'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y',
'source $HOME/.cargo/env',
'rustup default stable',
# This is missing from the Rocksdb installer (needed for Rocksdb).
'sudo apt-get install -y clang',
# Clone the repo.
f'(git clone {self.settings.repo_url} || (cd {self.settings.repo_name} ; git pull))'
]
hosts = self.manager.hosts(flat=True)
try:
g = Group(*hosts, user='******', connect_kwargs=self.connect)
g.run(' && '.join(cmd), hide=True)
Print.heading(f'Initialized testbed of {len(hosts)} nodes')
except (GroupException, ExecutionError) as e:
e = FabricError(e) if isinstance(e, GroupException) else e
raise BenchError('Failed to install repo on testbed', e)
def simple_command(self):
group = Group('localhost', '127.0.0.1')
result = group.run('echo foo', hide=True)
eq_(
[x.stdout.strip() for x in result.values()],
['foo', 'foo'],
)
def run_commands(connections: ThreadingGroup, commands_file) -> None:
"""
Run commands on each remote host and prefix the output (stdout + stderr)
with the hostname.
**Note about the host prefix**
Cf. http://www.fabfile.org/upgrading.html, the "env.output_prefix" option
has not been ported to fabric 2.X
We by-pass this by writing the current hostname to a file and we prefix
each output of the commands run with the hostname
:param connections:
:param commands_file:
:return:
"""
connection: Connection
for connection in connections:
connection.run('echo "{host}" > /tmp/evaneos_ssh__fabric_host'.format(
host=connection.host))
try:
connections.run(
'{commands} 2>&1 | sed "s/^/[$(cat /tmp/evaneos_ssh__fabric_host)] /"'
.format(commands=commands_file))
except GroupException:
sys.exit(1)
finally:
remote_cleanup(connections, commands_file)
def _config(self, hosts, node_parameters):
Print.info('Generating configuration files...')
# Cleanup all local configuration files.
cmd = CommandMaker.cleanup()
subprocess.run([cmd], shell=True, stderr=subprocess.DEVNULL)
# Recompile the latest code.
cmd = CommandMaker.compile().split()
subprocess.run(cmd, check=True, cwd=PathMaker.node_crate_path())
# Create alias for the client and nodes binary.
cmd = CommandMaker.alias_binaries(PathMaker.binary_path())
subprocess.run([cmd], shell=True)
# Generate configuration files.
keys = []
key_files = [PathMaker.key_file(i) for i in range(len(hosts))]
for filename in key_files:
cmd = CommandMaker.generate_key(filename).split()
subprocess.run(cmd, check=True)
keys += [Key.from_file(filename)]
# Generate threshold signature files.
nodes = len(hosts)
cmd = './node threshold_keys'
for i in range(nodes):
cmd += ' --filename ' + PathMaker.threshold_key_file(i)
cmd = cmd.split()
subprocess.run(cmd, capture_output=True, check=True)
names = [x.name for x in keys]
consensus_addr = [f'{x}:{self.settings.consensus_port}' for x in hosts]
front_addr = [f'{x}:{self.settings.front_port}' for x in hosts]
tss_keys = []
for i in range(nodes):
tss_keys += [TSSKey.from_file(PathMaker.threshold_key_file(i))]
ids = [x.id for x in tss_keys]
mempool_addr = [f'{x}:{self.settings.mempool_port}' for x in hosts]
committee = Committee(names, ids, consensus_addr, front_addr,
mempool_addr)
committee.print(PathMaker.committee_file())
node_parameters.print(PathMaker.parameters_file())
# Cleanup all nodes.
cmd = f'{CommandMaker.cleanup()} || true'
g = Group(*hosts, user='******', connect_kwargs=self.connect)
g.run(cmd, hide=True)
# Upload configuration files.
progress = progress_bar(hosts, prefix='Uploading config files:')
for i, host in enumerate(progress):
c = Connection(host, user='******', connect_kwargs=self.connect)
c.put(PathMaker.committee_file(), '.')
c.put(PathMaker.key_file(i), '.')
c.put(PathMaker.threshold_key_file(i), '.')
c.put(PathMaker.parameters_file(), '.')
return committee
def excepted_command(self):
group = Group('nopebadhost1', 'nopebadhost2')
try:
group.run('lolnope', hide=True)
except GroupException as e:
for value in e.result.values():
ok_(isinstance(value, gaierror))
else:
assert False, "Did not raise GroupException!"
def excepted_command(self):
group = Group("nopebadhost1", "nopebadhost2")
try:
group.run("lolnope", hide=True)
except GroupException as e:
for value in e.result.values():
assert isinstance(value, gaierror)
else:
assert False, "Did not raise GroupException!"
def excepted_command(self):
group = Group('nopebadhost1', 'nopebadhost2')
try:
group.run('lolnope', hide=True)
except GroupException as e:
for value in e.result.values():
assert isinstance(value, gaierror)
else:
assert False, "Did not raise GroupException!"
def remote_cleanup(connections: ThreadingGroup, commands_file: str) -> None:
"""
Cleans up remote hosts' filesystem.
:param connections: The remote connections
:param commands_file: The command file to delete
"""
connections.run('rm {file}'.format(file=commands_file))
connections.run(
'rm /tmp/evaneos_ssh__fabric_host'.format(file=commands_file))
def kill(self, hosts=[], delete_logs=False):
assert isinstance(hosts, list)
assert isinstance(delete_logs, bool)
hosts = hosts if hosts else self.manager.hosts(flat=True)
delete_logs = CommandMaker.clean_logs() if delete_logs else 'true'
cmd = [delete_logs, f'({CommandMaker.kill()} || true)']
try:
g = Group(*hosts, user='******', connect_kwargs=self.connect)
g.run(' && '.join(cmd), hide=True)
except GroupException as e:
raise BenchError('Failed to kill nodes', FabricError(e))
def failed_command(self):
group = Group('localhost', '127.0.0.1')
try:
group.run('lolnope', hide=True)
except GroupException as e:
# GroupException.result -> GroupResult;
# GroupResult values will be UnexpectedExit in this case;
# UnexpectedExit.result -> Result, and thus .exited etc.
exits = [x.result.exited for x in e.result.values()]
assert [127, 127] == exits
else:
assert False, "Did not raise GroupException!"
def failed_command(self):
group = Group("localhost", "127.0.0.1")
try:
group.run("lolnope", hide=True)
except GroupException as e:
# GroupException.result -> GroupResult;
# GroupResult values will be UnexpectedExit in this case;
# UnexpectedExit.result -> Result, and thus .exited etc.
exits = [x.result.exited for x in e.result.values()]
assert [127, 127] == exits
else:
assert False, "Did not raise GroupException!"
def failed_command(self):
group = Group('localhost', '127.0.0.1')
try:
group.run('lolnope', hide=True)
except GroupException as e:
# GroupException.result -> GroupResult;
# GroupResult values will be UnexpectedExit in this case;
# UnexpectedExit.result -> Result, and thus .exited etc.
eq_(
[x.result.exited for x in e.result.values()],
[127, 127],
)
else:
assert False, "Did not raise GroupException!"
def _update(self, hosts):
Print.info(
f'Updating {len(hosts)} nodes (branch "{self.settings.branch}")...'
)
cmd = [
f'(cd {self.settings.repo_name} && git fetch -f)',
f'(cd {self.settings.repo_name} && git checkout -f {self.settings.branch})',
f'(cd {self.settings.repo_name} && git pull -f)',
'source $HOME/.cargo/env',
f'(cd {self.settings.repo_name}/node && {CommandMaker.compile()})',
CommandMaker.alias_binaries(
f'./{self.settings.repo_name}/target/release/')
]
g = Group(*hosts, user='******', connect_kwargs=self.connect)
g.run(' && '.join(cmd), hide=True)
def run_one2all(bramble, client_ip, niter=niter):
client = Connection(client_ip, user='******', connect_kwargs=cxn_args)
ips = [c.host for c in bramble if c.host != client_ip]
servers = ThreadingGroup(*ips, user='******', connect_kwargs=cxn_args)
print(f"Begin 1 client to {len(servers)} servers experiment")
for c in servers:
c.run("killall -q iperf", warn=True)
time.sleep(10) # wait for old process to die
c.run(f"iperf -s > {remote_output_dir}/server.log &")
# I'm just *creating* the command here, not running it yet
ips = [c.host for c in servers]
cmds = [
f"(iperf -P 2 -c {s} >> {remote_output_dir}/client-to-s{i+1}.log &)"
for i, s in enumerate(ips)
]
cmd = ';'.join(cmds)
print("one2all command string: ", cmd)
for i in range(niter):
print(f"Iteration {i}")
client.run("killall -q iperf", warn=True)
time.sleep(10) # wait for the processes to die
client.run(cmd)
gather_one2all_results(client, servers)
def bubbles_up_errors_within_threads(self):
# TODO: I feel like this is the first spot where a raw
# ThreadException might need tweaks, at least presentation-wise,
# since we're no longer dealing with truly background threads (IO
# workers and tunnels), but "middle-ground" threads the user is
# kind of expecting (and which they might expect to encounter
# failures).
cxns = [Mock(host=x) for x in ("host1", "host2", "host3")]
class OhNoz(Exception):
pass
onoz = OhNoz()
cxns[1].run.side_effect = onoz
g = ThreadingGroup.from_connections(cxns)
try:
g.run(*self.args, **self.kwargs)
except GroupException as e:
result = e.result
else:
assert False, "Did not raise GroupException!"
succeeded = {
cxns[0]: cxns[0].run.return_value,
cxns[2]: cxns[2].run.return_value,
}
failed = {cxns[1]: onoz}
expected = succeeded.copy()
expected.update(failed)
assert result == expected
assert result.succeeded == succeeded
assert result.failed == failed
def executes_arguments_on_contents_run_via_threading(
self,
Thread,
Queue,
):
queue = Queue.return_value
g = ThreadingGroup.from_connections(self.cxns)
# Make sure .exception() doesn't yield truthy Mocks. Otherwise we
# end up with 'exceptions' that cause errors due to all being the
# same.
Thread.return_value.exception.return_value = None
g.run(*self.args, **self.kwargs)
# Testing that threads were used the way we expect is mediocre but
# I honestly can't think of another good way to assert "threading
# was used & concurrency occurred"...
instantiations = [
call(
target=thread_worker,
kwargs=dict(
cxn=cxn,
queue=queue,
args=self.args,
kwargs=self.kwargs,
),
) for cxn in self.cxns
]
Thread.assert_has_calls(instantiations, any_order=True)
# These ought to work as by default a Mock.return_value is a
# singleton mock object
expected = len(self.cxns)
for name, got in (('start', Thread.return_value.start.call_count),
('join', Thread.return_value.join.call_count)):
err = "Expected {} calls to ExceptionHandlingThread.{}, got {}" # noqa
err = err.format(expected, name, got)
assert expected, got == err
def bubbles_up_errors_within_threads(self):
# TODO: I feel like this is the first spot where a raw
# ThreadException might need tweaks, at least presentation-wise,
# since we're no longer dealing with truly background threads (IO
# workers and tunnels), but "middle-ground" threads the user is
# kind of expecting (and which they might expect to encounter
# failures).
cxns = [Mock(host=x) for x in ("host1", "host2", "host3")]
class OhNoz(Exception):
pass
onoz = OhNoz()
cxns[1].run.side_effect = onoz
g = ThreadingGroup.from_connections(cxns)
try:
g.run(*self.args, **self.kwargs)
except GroupException as e:
result = e.result
else:
assert False, "Did not raise GroupException!"
succeeded = {
cxns[0]: cxns[0].run.return_value,
cxns[2]: cxns[2].run.return_value,
}
failed = {cxns[1]: onoz}
expected = succeeded.copy()
expected.update(failed)
assert result == expected
assert result.succeeded == succeeded
assert result.failed == failed
def run_servers(conns):
commands = {}
for node_index, _ in enumerate(conns):
program_cmd = make_batch_cmd([
f"killall -9 {exe_name}",
"sleep 1",
f"nohup script -c './sharding-poc -seed={node_index} 2>&1 1>poc_{node_index}.out &' /dev/null",
])
node_cmd = make_and_cmd([
"cd {}".format(make_repo_src_path(poc_repo)),
program_cmd,
])
# node_cmd = "nohup sleep 20 > 123.txt < /dev/null &"
# node_cmd = "cd {}".format(make_repo_src_path(poc_repo))
# node_cmd += f"&& nohup ./{exe_name} -seed={node_index} > poc_node_{node_index}.out < /dev/null & "
# node_cmd = "nohup ./sharding-poc -seed=0 /dev/null &"
# node_cmd = "(nohup ./sharding-poc -seed 0 &) && ps aux|grep sharding-poc"
# node_cmd = "setsid ./sharding-poc -seed 0"
# node_cmd = "python3 ./temp.py > haha.txt"
# node_cmd = "nohup ./c_sleep > haha.txt &"
commands[node_index] = node_cmd
print(commands)
g = ThreadingGroup.from_connections(node_conns).run(custom_kwargs=commands)
print(g)
def multipleHosts(self):
servers = [
Connection("%s@%s" % (host['username'], host['hostname']),
connect_kwargs={"password": host['password']})
for host in self.servers_list
]
connections = ThreadingGroup.from_connections(servers)
return connections
def returns_results_mapping(self, method):
cxns = [Mock(name=x) for x in ("host1", "host2", "host3")]
g = ThreadingGroup.from_connections(cxns)
result = getattr(g, method)("whatever", hide=True)
assert isinstance(result, GroupResult)
expected = {x: getattr(x, method).return_value for x in cxns}
assert result == expected
assert result.succeeded == expected
assert result.failed == {}
def configure_group(hosts):
connections = []
for host in hosts:
connections.append(
Connection(host=host,
user=user,
connect_kwargs={'key_filename': 'C:\\Users\\Evan.Evan-Desktop\\.ssh\\dht-aws-key.pem'}))
group = Group.from_connections(connections)
return group
def returns_results_mapping(self):
# TODO: update if/when we implement ResultSet
cxns = [Mock(name=x) for x in ("host1", "host2", "host3")]
g = ThreadingGroup.from_connections(cxns)
result = g.run("whatever", hide=True)
assert isinstance(result, GroupResult)
expected = {x: x.run.return_value for x in cxns}
assert result == expected
assert result.succeeded == expected
assert result.failed == {}
def upload_command_file_to_remotes(connections: ThreadingGroup) -> str:
"""
Upload the command file to execute remotely on each remote host.
:param connections: The remote connections
:return: The remote path of the command file
"""
remote_commands_file = '/tmp/{base_filename}.sh'.format(
base_filename=str(uuid.uuid4()))
connection: Connection
for connection in connections:
connection.put('/var/local/github-actions/commands',
remote_commands_file)
connections.run('chmod a+x {file}'.format(file=remote_commands_file))
return remote_commands_file
def returns_results_mapping(self):
# TODO: update if/when we implement ResultSet
cxns = [Mock(name=x) for x in ("host1", "host2", "host3")]
g = ThreadingGroup.from_connections(cxns)
result = g.run("whatever", hide=True)
assert isinstance(result, GroupResult)
expected = {x: x.run.return_value for x in cxns}
assert result == expected
assert result.succeeded == expected
assert result.failed == {}
def test_hosts_are_reachable(connections_to_test: ThreadingGroup) -> None:
"""
Run a dummy command on remote host to make sure they are reachable.
If an issue arise at connection, display the exception for each connection in error.
:param connections_to_test: The connections we make sure are reachable
"""
try:
connections_to_test.run('echo "dry-run"', hide=True)
except GroupException as error:
result: Connection
for error_connection, error in error.result.items():
if isinstance(error, Exception):
print_github_action_error(
'Error on "{host}": {message}'.format(
host=error_connection.host, message=str(error)))
sys.exit(1)
def group(self, thread=True, conns=None, **kwargs):
group = ThreadingGroup() if thread else Group()
if conns is not None:
group.extend(conns)
else:
group.extend(self.conns(**kwargs))
return group
def run_test():
n_hosts = os.environ.get('PYINFRA_TEST_HOSTS', '5')
n_hosts = int(n_hosts)
hosts = [
'root@localhost:{0}'.format(9000 + n)
for n in range(0, n_hosts)
]
hosts = ThreadingGroup(*hosts, connect_kwargs={
'key_filename': 'docker/performance_rsa',
})
add_pyinfra_user(hosts)
add_log_file(hosts)
copy_a_file(hosts)
run_some_shell(hosts)
def broadcastcollation(conns):
commands = {}
exact_conns = []
for node_index, conn in enumerate(conns):
program_cmd = make_batch_cmd(
[f"./{exe_name} -seed={node_index} -client "])
if node_index not in node_send_collation:
continue
program_cmd += "broadcastcollation {}".format(" ".join(
map(str, node_send_collation[node_index])))
node_cmd = make_and_cmd([
"cd {}".format(make_repo_src_path(poc_repo)),
program_cmd,
])
exact_conns.append(conn)
commands[node_index] = node_cmd
print(commands)
g = ThreadingGroup.from_connections(exact_conns).run(
custom_kwargs=commands)
print(g)
def update_build_poc(conns):
g = ThreadingGroup.from_connections(conns).run(
make_batch_cmd([
make_cmd_setup(libp2p_repo),
make_cmd_pull(libp2p_repo),
make_cmd_setup(pubsub_repo),
make_cmd_pull(pubsub_repo),
make_cmd_setup(poc_repo),
make_cmd_pull(poc_repo),
make_cmd_build(poc_repo),
]))
for conn in conns:
result = g[conn]
if not result.ok:
raise ValueError(
"building failed in conn {}, stdout=\"{}\", stderr=\"{}\"".
format(
conn,
result.stdout,
result.stderr,
))
def queue_used_to_return_results(self, Queue):
# Regular, explicit, mocks for Connections
cxns = [Mock(host=x) for x in ("host1", "host2", "host3")]
# Set up Queue with enough behavior to work / assert
queue = Queue.return_value
# Ending w/ a True will terminate a while-not-empty loop
queue.empty.side_effect = (False, False, False, True)
fakes = [(x, x.run.return_value) for x in cxns]
queue.get.side_effect = fakes[:]
# Execute & inspect results
g = ThreadingGroup.from_connections(cxns)
results = g.run(*self.args, **self.kwargs)
expected = {x: x.run.return_value for x in cxns}
assert results == expected
# Make sure queue was used as expected within worker &
# ThreadingGroup.run()
puts = [call(x) for x in fakes]
queue.put.assert_has_calls(puts, any_order=True)
assert queue.empty.called
gets = [call(block=False) for _ in cxns]
queue.get.assert_has_calls(gets)
def queue_used_to_return_results(self, Queue):
# Regular, explicit, mocks for Connections
cxns = [Mock(host=x) for x in ("host1", "host2", "host3")]
# Set up Queue with enough behavior to work / assert
queue = Queue.return_value
# Ending w/ a True will terminate a while-not-empty loop
queue.empty.side_effect = (False, False, False, True)
fakes = [(x, x.run.return_value) for x in cxns]
queue.get.side_effect = fakes[:]
# Execute & inspect results
g = ThreadingGroup.from_connections(cxns)
results = g.run(*self.args, **self.kwargs)
expected = {x: x.run.return_value for x in cxns}
assert results == expected
# Make sure queue was used as expected within worker &
# ThreadingGroup.run()
puts = [call(x) for x in fakes]
queue.put.assert_has_calls(puts, any_order=True)
assert queue.empty.called
gets = [call(block=False) for _ in cxns]
queue.get.assert_has_calls(gets)
def __init__(self,
user,
hosts,
max_workers=100,
submission_root=None,
prolog="",
rc=None):
"""
"""
if not isinstance(hosts, dict):
hosts = {host: {} for host in hosts}
self.user = user
self.max_workers = max_workers
self.prolog = prolog
self.hosts = hosts
self.submission_root = submission_root
self.rc = rc
self.connections = Group(*list(sorted(hosts.keys())), user=user)
for connection in self.connections:
connection.connect_timeout = 360
def executes_arguments_on_contents_run_via_threading(
self, Thread, Queue
):
queue = Queue.return_value
g = ThreadingGroup.from_connections(self.cxns)
# Make sure .exception() doesn't yield truthy Mocks. Otherwise we
# end up with 'exceptions' that cause errors due to all being the
# same.
Thread.return_value.exception.return_value = None
g.run(*self.args, **self.kwargs)
# Testing that threads were used the way we expect is mediocre but
# I honestly can't think of another good way to assert "threading
# was used & concurrency occurred"...
instantiations = [
call(
target=thread_worker,
kwargs=dict(
cxn=cxn,
queue=queue,
args=self.args,
kwargs=self.kwargs,
),
)
for cxn in self.cxns
]
Thread.assert_has_calls(instantiations, any_order=True)
# These ought to work as by default a Mock.return_value is a
# singleton mock object
expected = len(self.cxns)
for name, got in (
("start", Thread.return_value.start.call_count),
("join", Thread.return_value.join.call_count),
):
err = (
"Expected {} calls to ExceptionHandlingThread.{}, got {}"
) # noqa
err = err.format(expected, name, got)
assert expected, got == err
def run_servers(conns):
commands = {}
for node_index, _ in enumerate(conns):
linux_command = f"script -f -c './{exe_name} -seed={node_index}' poc_{node_index}.out"
osx_command = f"script -q /dev/null ./{exe_name} -seed={node_index} 2>&1 1>poc_{node_index}.out /dev/null"
program_cmd = make_batch_cmd([
f"killall -9 {exe_name}",
"sleep 1",
f"screen -d -m bash -c \"if [ \"$(uname)\" == \"Darwin\" ]; then {osx_command}; else {linux_command}; fi\"",
])
node_cmd = make_and_cmd([
cmd_set_env,
"cd {}".format(make_repo_src_path(poc_repo)),
program_cmd,
])
commands[node_index] = node_cmd
g = ThreadingGroup.from_connections(node_conns).run(custom_kwargs=commands,
echo=True)
print(g)
def addpeer(conns):
commands = {}
exact_conns = []
for node_index, conn in enumerate(conns):
program_cmd = f"./{exe_name} -seed={node_index} -client "
if node_index not in node_target:
continue
target_node_index = node_target[node_index]
target_node_host_index = node_host_index_map[target_node_index]
program_cmd += "addpeer {} {} ".format(
hosts[target_node_host_index].ip,
target_node_index,
)
node_cmd = make_and_cmd([
"cd {}".format(make_repo_src_path(poc_repo)),
program_cmd,
])
exact_conns.append(conn)
commands[node_index] = node_cmd
print(commands)
g = ThreadingGroup.from_connections(exact_conns).run(
custom_kwargs=commands)
print(g)
def run_all2one(bramble, server_ip, niter=niter):
server = Connection(server_ip, user='******', connect_kwargs=cxn_args)
ips = [c.host for c in bramble if c.host != server_ip]
clients = ThreadingGroup(*ips, user='******', connect_kwargs=cxn_args)
print(f"Begin {len(clients)} clients to 1 server experiment")
server.run("killall -q iperf", warn=True)
time.sleep(10) # wait for old process to die
server.run(f"iperf -s > {remote_output_dir}/server.log &")
for i in range(niter):
print(f"Iteration {i}")
clients.run("killall -q iperf", warn=True)
time.sleep(10) # wait for processes to die
clients.run(
f"iperf -P 20 -c {server.host} >> {remote_output_dir}/client.log")
gather_all2one_results(clients, server)
def simple_command(self):
group = Group("localhost", "127.0.0.1")
result = group.run("echo foo", hide=True)
outs = [x.stdout.strip() for x in result.values()]
assert ["foo", "foo"] == outs
def main(hosts, concurrency):
with ThreadingGroup(*hosts) as sg:
results = sg.run('/home/claw/anaconda3/bin/activate py36 && '
f'celery multi start w1 -A scheduler '
f'--concurrency={concurrency} '
f'-l info -n %h --pidfile=/var/run/celery/%n.pid --logfile=/var/log/celery/%n%I.log')
# coding: utf-8
from fabric import ThreadingGroup
from ssh.github_actions.input import parse_input
from ssh.ssh_utils.command import upload_command_file_to_remotes, run_commands
from ssh.ssh_utils.test_reachable import test_hosts_are_reachable
if __name__ == '__main__':
action_input = parse_input()
connection_config = {}
if action_input.password is not None:
connection_config = {
'password': action_input.password
}
connections = ThreadingGroup(
*action_input.hostnames,
user=action_input.user,
port=action_input.port,
connect_timeout=20,
connect_kwargs=connection_config
)
test_hosts_are_reachable(connections)
remote_command_file = upload_command_file_to_remotes(connections)
run_commands(connections, remote_command_file)
