def install(self): Print.info('Installing rust and cloning the repo...') cmd = [ 'sudo apt-get update', 'sudo apt-get -y upgrade', 'sudo apt-get -y autoremove', # The following dependencies prevent the error: [error: linker `cc` not found]. 'sudo apt-get -y install build-essential', 'sudo apt-get -y install cmake', # Install rust (non-interactive). 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y', 'source $HOME/.cargo/env', 'rustup default stable', # This is missing from the Rocksdb installer (needed for Rocksdb). 'sudo apt-get install -y clang', # Clone the repo. f'(git clone {self.settings.repo_url} || (cd {self.settings.repo_name} ; git pull))' ] hosts = self.manager.hosts(flat=True) try: g = Group(*hosts, user='******', connect_kwargs=self.connect) g.run(' && '.join(cmd), hide=True) Print.heading(f'Initialized testbed of {len(hosts)} nodes') except (GroupException, ExecutionError) as e: e = FabricError(e) if isinstance(e, GroupException) else e raise BenchError('Failed to install repo on testbed', e)
def simple_command(self): group = Group('localhost', '127.0.0.1') result = group.run('echo foo', hide=True) eq_( [x.stdout.strip() for x in result.values()], ['foo', 'foo'], )
def run_commands(connections: ThreadingGroup, commands_file) -> None: """ Run commands on each remote host and prefix the output (stdout + stderr) with the hostname. **Note about the host prefix** Cf. http://www.fabfile.org/upgrading.html, the "env.output_prefix" option has not been ported to fabric 2.X We by-pass this by writing the current hostname to a file and we prefix each output of the commands run with the hostname :param connections: :param commands_file: :return: """ connection: Connection for connection in connections: connection.run('echo "{host}" > /tmp/evaneos_ssh__fabric_host'.format( host=connection.host)) try: connections.run( '{commands} 2>&1 | sed "s/^/[$(cat /tmp/evaneos_ssh__fabric_host)] /"' .format(commands=commands_file)) except GroupException: sys.exit(1) finally: remote_cleanup(connections, commands_file)
def _config(self, hosts, node_parameters): Print.info('Generating configuration files...') # Cleanup all local configuration files. cmd = CommandMaker.cleanup() subprocess.run([cmd], shell=True, stderr=subprocess.DEVNULL) # Recompile the latest code. cmd = CommandMaker.compile().split() subprocess.run(cmd, check=True, cwd=PathMaker.node_crate_path()) # Create alias for the client and nodes binary. cmd = CommandMaker.alias_binaries(PathMaker.binary_path()) subprocess.run([cmd], shell=True) # Generate configuration files. keys = [] key_files = [PathMaker.key_file(i) for i in range(len(hosts))] for filename in key_files: cmd = CommandMaker.generate_key(filename).split() subprocess.run(cmd, check=True) keys += [Key.from_file(filename)] # Generate threshold signature files. nodes = len(hosts) cmd = './node threshold_keys' for i in range(nodes): cmd += ' --filename ' + PathMaker.threshold_key_file(i) cmd = cmd.split() subprocess.run(cmd, capture_output=True, check=True) names = [x.name for x in keys] consensus_addr = [f'{x}:{self.settings.consensus_port}' for x in hosts] front_addr = [f'{x}:{self.settings.front_port}' for x in hosts] tss_keys = [] for i in range(nodes): tss_keys += [TSSKey.from_file(PathMaker.threshold_key_file(i))] ids = [x.id for x in tss_keys] mempool_addr = [f'{x}:{self.settings.mempool_port}' for x in hosts] committee = Committee(names, ids, consensus_addr, front_addr, mempool_addr) committee.print(PathMaker.committee_file()) node_parameters.print(PathMaker.parameters_file()) # Cleanup all nodes. cmd = f'{CommandMaker.cleanup()} || true' g = Group(*hosts, user='******', connect_kwargs=self.connect) g.run(cmd, hide=True) # Upload configuration files. progress = progress_bar(hosts, prefix='Uploading config files:') for i, host in enumerate(progress): c = Connection(host, user='******', connect_kwargs=self.connect) c.put(PathMaker.committee_file(), '.') c.put(PathMaker.key_file(i), '.') c.put(PathMaker.threshold_key_file(i), '.') c.put(PathMaker.parameters_file(), '.') return committee
def excepted_command(self): group = Group('nopebadhost1', 'nopebadhost2') try: group.run('lolnope', hide=True) except GroupException as e: for value in e.result.values(): ok_(isinstance(value, gaierror)) else: assert False, "Did not raise GroupException!"
def excepted_command(self): group = Group("nopebadhost1", "nopebadhost2") try: group.run("lolnope", hide=True) except GroupException as e: for value in e.result.values(): assert isinstance(value, gaierror) else: assert False, "Did not raise GroupException!"
def excepted_command(self): group = Group('nopebadhost1', 'nopebadhost2') try: group.run('lolnope', hide=True) except GroupException as e: for value in e.result.values(): assert isinstance(value, gaierror) else: assert False, "Did not raise GroupException!"
def remote_cleanup(connections: ThreadingGroup, commands_file: str) -> None: """ Cleans up remote hosts' filesystem. :param connections: The remote connections :param commands_file: The command file to delete """ connections.run('rm {file}'.format(file=commands_file)) connections.run( 'rm /tmp/evaneos_ssh__fabric_host'.format(file=commands_file))
def kill(self, hosts=[], delete_logs=False): assert isinstance(hosts, list) assert isinstance(delete_logs, bool) hosts = hosts if hosts else self.manager.hosts(flat=True) delete_logs = CommandMaker.clean_logs() if delete_logs else 'true' cmd = [delete_logs, f'({CommandMaker.kill()} || true)'] try: g = Group(*hosts, user='******', connect_kwargs=self.connect) g.run(' && '.join(cmd), hide=True) except GroupException as e: raise BenchError('Failed to kill nodes', FabricError(e))
def failed_command(self): group = Group('localhost', '127.0.0.1') try: group.run('lolnope', hide=True) except GroupException as e: # GroupException.result -> GroupResult; # GroupResult values will be UnexpectedExit in this case; # UnexpectedExit.result -> Result, and thus .exited etc. exits = [x.result.exited for x in e.result.values()] assert [127, 127] == exits else: assert False, "Did not raise GroupException!"
def failed_command(self): group = Group("localhost", "127.0.0.1") try: group.run("lolnope", hide=True) except GroupException as e: # GroupException.result -> GroupResult; # GroupResult values will be UnexpectedExit in this case; # UnexpectedExit.result -> Result, and thus .exited etc. exits = [x.result.exited for x in e.result.values()] assert [127, 127] == exits else: assert False, "Did not raise GroupException!"
def failed_command(self): group = Group('localhost', '127.0.0.1') try: group.run('lolnope', hide=True) except GroupException as e: # GroupException.result -> GroupResult; # GroupResult values will be UnexpectedExit in this case; # UnexpectedExit.result -> Result, and thus .exited etc. eq_( [x.result.exited for x in e.result.values()], [127, 127], ) else: assert False, "Did not raise GroupException!"
def _update(self, hosts): Print.info( f'Updating {len(hosts)} nodes (branch "{self.settings.branch}")...' ) cmd = [ f'(cd {self.settings.repo_name} && git fetch -f)', f'(cd {self.settings.repo_name} && git checkout -f {self.settings.branch})', f'(cd {self.settings.repo_name} && git pull -f)', 'source $HOME/.cargo/env', f'(cd {self.settings.repo_name}/node && {CommandMaker.compile()})', CommandMaker.alias_binaries( f'./{self.settings.repo_name}/target/release/') ] g = Group(*hosts, user='******', connect_kwargs=self.connect) g.run(' && '.join(cmd), hide=True)
def run_one2all(bramble, client_ip, niter=niter): client = Connection(client_ip, user='******', connect_kwargs=cxn_args) ips = [c.host for c in bramble if c.host != client_ip] servers = ThreadingGroup(*ips, user='******', connect_kwargs=cxn_args) print(f"Begin 1 client to {len(servers)} servers experiment") for c in servers: c.run("killall -q iperf", warn=True) time.sleep(10) # wait for old process to die c.run(f"iperf -s > {remote_output_dir}/server.log &") # I'm just *creating* the command here, not running it yet ips = [c.host for c in servers] cmds = [ f"(iperf -P 2 -c {s} >> {remote_output_dir}/client-to-s{i+1}.log &)" for i, s in enumerate(ips) ] cmd = ';'.join(cmds) print("one2all command string: ", cmd) for i in range(niter): print(f"Iteration {i}") client.run("killall -q iperf", warn=True) time.sleep(10) # wait for the processes to die client.run(cmd) gather_one2all_results(client, servers)
def bubbles_up_errors_within_threads(self): # TODO: I feel like this is the first spot where a raw # ThreadException might need tweaks, at least presentation-wise, # since we're no longer dealing with truly background threads (IO # workers and tunnels), but "middle-ground" threads the user is # kind of expecting (and which they might expect to encounter # failures). cxns = [Mock(host=x) for x in ("host1", "host2", "host3")] class OhNoz(Exception): pass onoz = OhNoz() cxns[1].run.side_effect = onoz g = ThreadingGroup.from_connections(cxns) try: g.run(*self.args, **self.kwargs) except GroupException as e: result = e.result else: assert False, "Did not raise GroupException!" succeeded = { cxns[0]: cxns[0].run.return_value, cxns[2]: cxns[2].run.return_value, } failed = {cxns[1]: onoz} expected = succeeded.copy() expected.update(failed) assert result == expected assert result.succeeded == succeeded assert result.failed == failed
def executes_arguments_on_contents_run_via_threading( self, Thread, Queue, ): queue = Queue.return_value g = ThreadingGroup.from_connections(self.cxns) # Make sure .exception() doesn't yield truthy Mocks. Otherwise we # end up with 'exceptions' that cause errors due to all being the # same. Thread.return_value.exception.return_value = None g.run(*self.args, **self.kwargs) # Testing that threads were used the way we expect is mediocre but # I honestly can't think of another good way to assert "threading # was used & concurrency occurred"... instantiations = [ call( target=thread_worker, kwargs=dict( cxn=cxn, queue=queue, args=self.args, kwargs=self.kwargs, ), ) for cxn in self.cxns ] Thread.assert_has_calls(instantiations, any_order=True) # These ought to work as by default a Mock.return_value is a # singleton mock object expected = len(self.cxns) for name, got in (('start', Thread.return_value.start.call_count), ('join', Thread.return_value.join.call_count)): err = "Expected {} calls to ExceptionHandlingThread.{}, got {}" # noqa err = err.format(expected, name, got) assert expected, got == err
def run_servers(conns): commands = {} for node_index, _ in enumerate(conns): program_cmd = make_batch_cmd([ f"killall -9 {exe_name}", "sleep 1", f"nohup script -c './sharding-poc -seed={node_index} 2>&1 1>poc_{node_index}.out &' /dev/null", ]) node_cmd = make_and_cmd([ "cd {}".format(make_repo_src_path(poc_repo)), program_cmd, ]) # node_cmd = "nohup sleep 20 > 123.txt < /dev/null &" # node_cmd = "cd {}".format(make_repo_src_path(poc_repo)) # node_cmd += f"&& nohup ./{exe_name} -seed={node_index} > poc_node_{node_index}.out < /dev/null & " # node_cmd = "nohup ./sharding-poc -seed=0 /dev/null &" # node_cmd = "(nohup ./sharding-poc -seed 0 &) && ps aux|grep sharding-poc" # node_cmd = "setsid ./sharding-poc -seed 0" # node_cmd = "python3 ./temp.py > haha.txt" # node_cmd = "nohup ./c_sleep > haha.txt &" commands[node_index] = node_cmd print(commands) g = ThreadingGroup.from_connections(node_conns).run(custom_kwargs=commands) print(g)
def multipleHosts(self): servers = [ Connection("%s@%s" % (host['username'], host['hostname']), connect_kwargs={"password": host['password']}) for host in self.servers_list ] connections = ThreadingGroup.from_connections(servers) return connections
def returns_results_mapping(self, method): cxns = [Mock(name=x) for x in ("host1", "host2", "host3")] g = ThreadingGroup.from_connections(cxns) result = getattr(g, method)("whatever", hide=True) assert isinstance(result, GroupResult) expected = {x: getattr(x, method).return_value for x in cxns} assert result == expected assert result.succeeded == expected assert result.failed == {}
def configure_group(hosts): connections = [] for host in hosts: connections.append( Connection(host=host, user=user, connect_kwargs={'key_filename': 'C:\\Users\\Evan.Evan-Desktop\\.ssh\\dht-aws-key.pem'})) group = Group.from_connections(connections) return group
def returns_results_mapping(self): # TODO: update if/when we implement ResultSet cxns = [Mock(name=x) for x in ("host1", "host2", "host3")] g = ThreadingGroup.from_connections(cxns) result = g.run("whatever", hide=True) assert isinstance(result, GroupResult) expected = {x: x.run.return_value for x in cxns} assert result == expected assert result.succeeded == expected assert result.failed == {}
def upload_command_file_to_remotes(connections: ThreadingGroup) -> str: """ Upload the command file to execute remotely on each remote host. :param connections: The remote connections :return: The remote path of the command file """ remote_commands_file = '/tmp/{base_filename}.sh'.format( base_filename=str(uuid.uuid4())) connection: Connection for connection in connections: connection.put('/var/local/github-actions/commands', remote_commands_file) connections.run('chmod a+x {file}'.format(file=remote_commands_file)) return remote_commands_file
def test_hosts_are_reachable(connections_to_test: ThreadingGroup) -> None: """ Run a dummy command on remote host to make sure they are reachable. If an issue arise at connection, display the exception for each connection in error. :param connections_to_test: The connections we make sure are reachable """ try: connections_to_test.run('echo "dry-run"', hide=True) except GroupException as error: result: Connection for error_connection, error in error.result.items(): if isinstance(error, Exception): print_github_action_error( 'Error on "{host}": {message}'.format( host=error_connection.host, message=str(error))) sys.exit(1)
def group(self, thread=True, conns=None, **kwargs): group = ThreadingGroup() if thread else Group() if conns is not None: group.extend(conns) else: group.extend(self.conns(**kwargs)) return group
def run_test(): n_hosts = os.environ.get('PYINFRA_TEST_HOSTS', '5') n_hosts = int(n_hosts) hosts = [ 'root@localhost:{0}'.format(9000 + n) for n in range(0, n_hosts) ] hosts = ThreadingGroup(*hosts, connect_kwargs={ 'key_filename': 'docker/performance_rsa', }) add_pyinfra_user(hosts) add_log_file(hosts) copy_a_file(hosts) run_some_shell(hosts)
def broadcastcollation(conns): commands = {} exact_conns = [] for node_index, conn in enumerate(conns): program_cmd = make_batch_cmd( [f"./{exe_name} -seed={node_index} -client "]) if node_index not in node_send_collation: continue program_cmd += "broadcastcollation {}".format(" ".join( map(str, node_send_collation[node_index]))) node_cmd = make_and_cmd([ "cd {}".format(make_repo_src_path(poc_repo)), program_cmd, ]) exact_conns.append(conn) commands[node_index] = node_cmd print(commands) g = ThreadingGroup.from_connections(exact_conns).run( custom_kwargs=commands) print(g)
def update_build_poc(conns): g = ThreadingGroup.from_connections(conns).run( make_batch_cmd([ make_cmd_setup(libp2p_repo), make_cmd_pull(libp2p_repo), make_cmd_setup(pubsub_repo), make_cmd_pull(pubsub_repo), make_cmd_setup(poc_repo), make_cmd_pull(poc_repo), make_cmd_build(poc_repo), ])) for conn in conns: result = g[conn] if not result.ok: raise ValueError( "building failed in conn {}, stdout=\"{}\", stderr=\"{}\"". format( conn, result.stdout, result.stderr, ))
def queue_used_to_return_results(self, Queue): # Regular, explicit, mocks for Connections cxns = [Mock(host=x) for x in ("host1", "host2", "host3")] # Set up Queue with enough behavior to work / assert queue = Queue.return_value # Ending w/ a True will terminate a while-not-empty loop queue.empty.side_effect = (False, False, False, True) fakes = [(x, x.run.return_value) for x in cxns] queue.get.side_effect = fakes[:] # Execute & inspect results g = ThreadingGroup.from_connections(cxns) results = g.run(*self.args, **self.kwargs) expected = {x: x.run.return_value for x in cxns} assert results == expected # Make sure queue was used as expected within worker & # ThreadingGroup.run() puts = [call(x) for x in fakes] queue.put.assert_has_calls(puts, any_order=True) assert queue.empty.called gets = [call(block=False) for _ in cxns] queue.get.assert_has_calls(gets)
def __init__(self, user, hosts, max_workers=100, submission_root=None, prolog="", rc=None): """ """ if not isinstance(hosts, dict): hosts = {host: {} for host in hosts} self.user = user self.max_workers = max_workers self.prolog = prolog self.hosts = hosts self.submission_root = submission_root self.rc = rc self.connections = Group(*list(sorted(hosts.keys())), user=user) for connection in self.connections: connection.connect_timeout = 360
def executes_arguments_on_contents_run_via_threading( self, Thread, Queue ): queue = Queue.return_value g = ThreadingGroup.from_connections(self.cxns) # Make sure .exception() doesn't yield truthy Mocks. Otherwise we # end up with 'exceptions' that cause errors due to all being the # same. Thread.return_value.exception.return_value = None g.run(*self.args, **self.kwargs) # Testing that threads were used the way we expect is mediocre but # I honestly can't think of another good way to assert "threading # was used & concurrency occurred"... instantiations = [ call( target=thread_worker, kwargs=dict( cxn=cxn, queue=queue, args=self.args, kwargs=self.kwargs, ), ) for cxn in self.cxns ] Thread.assert_has_calls(instantiations, any_order=True) # These ought to work as by default a Mock.return_value is a # singleton mock object expected = len(self.cxns) for name, got in ( ("start", Thread.return_value.start.call_count), ("join", Thread.return_value.join.call_count), ): err = ( "Expected {} calls to ExceptionHandlingThread.{}, got {}" ) # noqa err = err.format(expected, name, got) assert expected, got == err
def run_servers(conns): commands = {} for node_index, _ in enumerate(conns): linux_command = f"script -f -c './{exe_name} -seed={node_index}' poc_{node_index}.out" osx_command = f"script -q /dev/null ./{exe_name} -seed={node_index} 2>&1 1>poc_{node_index}.out /dev/null" program_cmd = make_batch_cmd([ f"killall -9 {exe_name}", "sleep 1", f"screen -d -m bash -c \"if [ \"$(uname)\" == \"Darwin\" ]; then {osx_command}; else {linux_command}; fi\"", ]) node_cmd = make_and_cmd([ cmd_set_env, "cd {}".format(make_repo_src_path(poc_repo)), program_cmd, ]) commands[node_index] = node_cmd g = ThreadingGroup.from_connections(node_conns).run(custom_kwargs=commands, echo=True) print(g)
def addpeer(conns): commands = {} exact_conns = [] for node_index, conn in enumerate(conns): program_cmd = f"./{exe_name} -seed={node_index} -client " if node_index not in node_target: continue target_node_index = node_target[node_index] target_node_host_index = node_host_index_map[target_node_index] program_cmd += "addpeer {} {} ".format( hosts[target_node_host_index].ip, target_node_index, ) node_cmd = make_and_cmd([ "cd {}".format(make_repo_src_path(poc_repo)), program_cmd, ]) exact_conns.append(conn) commands[node_index] = node_cmd print(commands) g = ThreadingGroup.from_connections(exact_conns).run( custom_kwargs=commands) print(g)
def run_all2one(bramble, server_ip, niter=niter): server = Connection(server_ip, user='******', connect_kwargs=cxn_args) ips = [c.host for c in bramble if c.host != server_ip] clients = ThreadingGroup(*ips, user='******', connect_kwargs=cxn_args) print(f"Begin {len(clients)} clients to 1 server experiment") server.run("killall -q iperf", warn=True) time.sleep(10) # wait for old process to die server.run(f"iperf -s > {remote_output_dir}/server.log &") for i in range(niter): print(f"Iteration {i}") clients.run("killall -q iperf", warn=True) time.sleep(10) # wait for processes to die clients.run( f"iperf -P 20 -c {server.host} >> {remote_output_dir}/client.log") gather_all2one_results(clients, server)
def simple_command(self): group = Group("localhost", "127.0.0.1") result = group.run("echo foo", hide=True) outs = [x.stdout.strip() for x in result.values()] assert ["foo", "foo"] == outs
def main(hosts, concurrency): with ThreadingGroup(*hosts) as sg: results = sg.run('/home/claw/anaconda3/bin/activate py36 && ' f'celery multi start w1 -A scheduler ' f'--concurrency={concurrency} ' f'-l info -n %h --pidfile=/var/run/celery/%n.pid --logfile=/var/log/celery/%n%I.log')
# coding: utf-8 from fabric import ThreadingGroup from ssh.github_actions.input import parse_input from ssh.ssh_utils.command import upload_command_file_to_remotes, run_commands from ssh.ssh_utils.test_reachable import test_hosts_are_reachable if __name__ == '__main__': action_input = parse_input() connection_config = {} if action_input.password is not None: connection_config = { 'password': action_input.password } connections = ThreadingGroup( *action_input.hostnames, user=action_input.user, port=action_input.port, connect_timeout=20, connect_kwargs=connection_config ) test_hosts_are_reachable(connections) remote_command_file = upload_command_file_to_remotes(connections) run_commands(connections, remote_command_file)