def when_no_arg_or_kwarg_given(self):
g = Group("host1", "host2")
g._do = Mock()
g.get(remote="whatever")
g._do.assert_called_with("get",
remote="whatever",
local="{host}/")
def contextmanager_behavior_works_like_Connection(self):
cxns = [Mock(name=x) for x in ("foo", "bar", "biz")]
g = Group.from_connections(cxns)
with g as my_g:
assert my_g is g
for c in cxns:
c.close.assert_called_once_with()
def _config(self, hosts, node_parameters):
Print.info('Generating configuration files...')
# Cleanup all local configuration files.
cmd = CommandMaker.cleanup()
subprocess.run([cmd], shell=True, stderr=subprocess.DEVNULL)
# Recompile the latest code.
cmd = CommandMaker.compile().split()
subprocess.run(cmd, check=True, cwd=PathMaker.node_crate_path())
# Create alias for the client and nodes binary.
cmd = CommandMaker.alias_binaries(PathMaker.binary_path())
subprocess.run([cmd], shell=True)
# Generate configuration files.
keys = []
key_files = [PathMaker.key_file(i) for i in range(len(hosts))]
for filename in key_files:
cmd = CommandMaker.generate_key(filename).split()
subprocess.run(cmd, check=True)
keys += [Key.from_file(filename)]
# Generate threshold signature files.
nodes = len(hosts)
cmd = './node threshold_keys'
for i in range(nodes):
cmd += ' --filename ' + PathMaker.threshold_key_file(i)
cmd = cmd.split()
subprocess.run(cmd, capture_output=True, check=True)
names = [x.name for x in keys]
consensus_addr = [f'{x}:{self.settings.consensus_port}' for x in hosts]
front_addr = [f'{x}:{self.settings.front_port}' for x in hosts]
tss_keys = []
for i in range(nodes):
tss_keys += [TSSKey.from_file(PathMaker.threshold_key_file(i))]
ids = [x.id for x in tss_keys]
mempool_addr = [f'{x}:{self.settings.mempool_port}' for x in hosts]
committee = Committee(names, ids, consensus_addr, front_addr,
mempool_addr)
committee.print(PathMaker.committee_file())
node_parameters.print(PathMaker.parameters_file())
# Cleanup all nodes.
cmd = f'{CommandMaker.cleanup()} || true'
g = Group(*hosts, user='******', connect_kwargs=self.connect)
g.run(cmd, hide=True)
# Upload configuration files.
progress = progress_bar(hosts, prefix='Uploading config files:')
for i, host in enumerate(progress):
c = Connection(host, user='******', connect_kwargs=self.connect)
c.put(PathMaker.committee_file(), '.')
c.put(PathMaker.key_file(i), '.')
c.put(PathMaker.threshold_key_file(i), '.')
c.put(PathMaker.parameters_file(), '.')
return committee
def run_group(group: fabric.Group,
cmd: str,
timeout: float = base.DEFAULT_TIMEOUT, # TODO
print_output: bool = False,
allow_failures: bool = False):
if allow_failures is False:
result = group.run(command=cmd, hide=not print_output)
else:
try:
result = group.run(command=cmd, hide=not print_output)
except invoke.exceptions.UnexpectedExit as e:
result = e.result
return result
def simple_command(self):
group = Group('localhost', '127.0.0.1')
result = group.run('echo foo', hide=True)
eq_(
[x.stdout.strip() for x in result.values()],
['foo', 'foo'],
)
def acts_like_an_iterable_of_Connections(self):
g = Group("foo", "bar", "biz")
assert g[0].host == "foo"
assert g[-1].host == "biz"
assert len(g) == 3
for c in g:
assert isinstance(c, Connection)
def install(self):
Print.info('Installing rust and cloning the repo...')
cmd = [
'sudo apt-get update',
'sudo apt-get -y upgrade',
'sudo apt-get -y autoremove',
# The following dependencies prevent the error: [error: linker `cc` not found].
'sudo apt-get -y install build-essential',
'sudo apt-get -y install cmake',
# Install rust (non-interactive).
'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y',
'source $HOME/.cargo/env',
'rustup default stable',
# This is missing from the Rocksdb installer (needed for Rocksdb).
'sudo apt-get install -y clang',
# Clone the repo.
f'(git clone {self.settings.repo_url} || (cd {self.settings.repo_name} ; git pull))'
]
hosts = self.manager.hosts(flat=True)
try:
g = Group(*hosts, user='******', connect_kwargs=self.connect)
g.run(' && '.join(cmd), hide=True)
Print.heading(f'Initialized testbed of {len(hosts)} nodes')
except (GroupException, ExecutionError) as e:
e = FabricError(e) if isinstance(e, GroupException) else e
raise BenchError('Failed to install repo on testbed', e)
def acts_like_an_iterable_of_Connections(self):
g = Group('foo', 'bar', 'biz')
assert g[0].host == 'foo'
assert g[-1].host == 'biz'
assert len(g) == 3
for c in g:
assert isinstance(c, Connection)
def restart(
c,
bootstrap,
zk,
healthcheck_retries=40,
healthcheck_wait=30,
conf=None,
pre_stop=None,
pre_start=None,
):
"""Gracefully restart an instance."""
admin = _admin_client(bootstrap, conf)
if hasattr(c, "host"):
hosts = [c.host]
else:
hosts = [b.host for b in brokers(admin)]
assert hosts, "broker list should not be empty!"
logger.info("Restarting all %s", ", ".join([hostname(ip) for ip in hosts]))
input("Press Enter to continue...")
# connect_kwargs contains common SSH configs for the hosts in the group
hosts = Group(*hosts, connect_kwargs=c.connect_kwargs)
for index, b in enumerate(hosts):
logger.info("(%s/%s) Restarting %s", index + 1, len(hosts), hostname(b.host))
start = datetime.now()
_instance_restart(
b, admin, zk, healthcheck_retries, healthcheck_wait, pre_stop, pre_start
)
end = datetime.now()
logger.info("Instance restart took: %s", end - start)
def group(self, thread=True, conns=None, **kwargs):
group = ThreadingGroup() if thread else Group()
if conns is not None:
group.extend(conns)
else:
group.extend(self.conns(**kwargs))
return group
def contextmanager_behavior_works_like_Connection(self):
cxns = [Mock(name=x) for x in ("foo", "bar", "biz")]
g = Group.from_connections(cxns)
with g as my_g:
assert my_g is g
for c in cxns:
c.close.assert_called_once_with()
def takes_splat_kwargs_and_passes_them_to_Connections(self):
g = Group("foo", "bar", user="******", forward_agent=True)
assert g[0].host == "foo"
assert g[0].user == "admin"
assert g[0].forward_agent is True
assert g[1].host == "bar"
assert g[1].user == "admin"
assert g[1].forward_agent is True
def get_ca_host_group(ca_host=None):
with open('ca-hosts.json', 'r') as f:
ca_hosts = json.load(f)
if ca_host is not None:
ca_hosts = [x for x in ca_hosts if x.startswith(ca_host)]
return Group(*ca_hosts, user='******', connect_timeout=5)
def excepted_command(self):
group = Group('nopebadhost1', 'nopebadhost2')
try:
group.run('lolnope', hide=True)
except GroupException as e:
for value in e.result.values():
assert isinstance(value, gaierror)
else:
assert False, "Did not raise GroupException!"
def kill(self, hosts=[], delete_logs=False):
assert isinstance(hosts, list)
assert isinstance(delete_logs, bool)
hosts = hosts if hosts else self.manager.hosts(flat=True)
delete_logs = CommandMaker.clean_logs() if delete_logs else 'true'
cmd = [delete_logs, f'({CommandMaker.kill()} || true)']
try:
g = Group(*hosts, user='******', connect_kwargs=self.connect)
g.run(' && '.join(cmd), hide=True)
except GroupException as e:
raise BenchError('Failed to kill nodes', FabricError(e))
def _get_hosts(self, password=''):
"""Get hosts from configuration"""
hosts = []
# Get mapping ports
private_key = self.get_local_path(
os.path.join('private_key', self._config['key']))
for port in sorted(self._config['mapping'].keys()):
hosts.append('localhost:' + str(port))
print("user", self._config['username'], "key_filename", private_key)
return Group(*hosts,
user=self._config['username'],
connect_kwargs={"key_filename": private_key})
def failed_command(self):
group = Group('localhost', '127.0.0.1')
try:
group.run('lolnope', hide=True)
except GroupException as e:
# GroupException.result -> GroupResult;
# GroupResult values will be UnexpectedExit in this case;
# UnexpectedExit.result -> Result, and thus .exited etc.
exits = [x.result.exited for x in e.result.values()]
assert [127, 127] == exits
else:
assert False, "Did not raise GroupException!"
def _get_hosts(self, password=''):
private_key = self.get_local_path(
os.path.join('private_key', self._config['key']))
hosts = []
hdfs = self._config['hdfs']
# First is master
hosts.append(hdfs['master'] + ":22")
for name, ip in hdfs.items():
if name != 'master':
hosts.append(ip + ":22")
return Group(*hosts,
user=self._config['username'],
connect_kwargs={"key_filename": private_key})
def get_hosts(ctx, port_forward='portforward.yaml'):
hosts = []
# Get mapping ports
config_file = get_config(port_forward)
print("Loading", config_file)
ports = benedict.from_yaml(config_file)
private_key = get_config(os.path.join('private_key', ports['key']))
for port, ip in ports['mapping'].items():
hosts.append('localhost:' + str(port))
print("user", ports['username'], "key_filename", private_key)
return Group(*hosts,
user=ports['username'],
connect_kwargs={"key_filename": private_key})
def _update(self, hosts):
Print.info(
f'Updating {len(hosts)} nodes (branch "{self.settings.branch}")...'
)
cmd = [
f'(cd {self.settings.repo_name} && git fetch -f)',
f'(cd {self.settings.repo_name} && git checkout -f {self.settings.branch})',
f'(cd {self.settings.repo_name} && git pull -f)',
'source $HOME/.cargo/env',
f'(cd {self.settings.repo_name}/node && {CommandMaker.compile()})',
CommandMaker.alias_binaries(
f'./{self.settings.repo_name}/target/release/')
]
g = Group(*hosts, user='******', connect_kwargs=self.connect)
g.run(' && '.join(cmd), hide=True)
def _get_hosts(self, password=''):
hosts = []
# Get mapping ports
private_key = self.get_local_path(
os.path.join('private_key', self._config['key']))
spark_range = self._config['spark-port-range']
port_start = int(spark_range.split('-')[0])
port_end = int(spark_range.split('-')[1])
for port in sorted(self._config['mapping'].keys()):
if port >= port_start and port <= port_end:
hosts.append('localhost:' + str(port))
print("user", self._config['username'], "key_filename", private_key)
return Group(*hosts,
user=self._config['username'],
connect_kwargs={"key_filename": private_key})
def __init__(self,
user,
hosts,
max_workers=100,
submission_root=None,
prolog="",
rc=None):
"""
"""
if not isinstance(hosts, dict):
hosts = {host: {} for host in hosts}
self.user = user
self.max_workers = max_workers
self.prolog = prolog
self.hosts = hosts
self.submission_root = submission_root
self.rc = rc
self.connections = Group(*list(sorted(hosts.keys())), user=user)
for connection in self.connections:
connection.connect_timeout = 360
def setup_slave(slaves: list, passphrase: str, user: str, master: str) -> None:
connect_kwargs = {'passphrase': passphrase}
nodes = Group(*slaves, user=user, connect_kwargs=connect_kwargs)
home = get_home_path(nodes)[0]
# install(nodes) # Not calling because of Fabric bug
for node in nodes:
# redis conf
logfile = f'logfile "{home}/redis.log"'
slaveof = f'slaveof {master} 6379'
print(logfile, slaveof)
node.put('redis_configs/redis.conf', 'redis-stable/redis.conf')
node.run(f'echo {slaveof} >> redis-stable/redis.conf')
node.run(f'echo {logfile} >> redis-stable/redis.conf')
# sentinal conf
logfile = f'logfile "{home}/sentinel.log"'
sentinel_monitor = f'sentinel monitor mymaster {master} 6379 2'
print(logfile, sentinel_monitor)
node.put('redis_configs/sentinel.conf', 'redis-stable/sentinel.conf')
node.run(f'echo {logfile} >> redis-stable/sentinel.conf')
node.run(f"sed -i 's/placeholder-line/{sentinel_monitor}/g' redis-stable/sentinel.conf")
def restart(
c,
bootstrap,
zk,
healthcheck_retries=40,
healthcheck_wait=30,
conf=None,
pre_stop=None,
):
"""Gracefully restart an instance."""
admin = _admin_client(bootstrap, conf)
if hasattr(c, "host"):
hosts = [c.host]
else:
hosts = [b.host for b in brokers(admin)]
assert hosts, "broker list should not be empty!"
logger.info("Restarting all %s",
", ".join([hostname(ip) for ip in hosts]))
input("Press Enter to continue...")
# connect_kwargs contains common SSH configs for the hosts in the group
for b in Group(*hosts, connect_kwargs=c.connect_kwargs):
logger.info("Restarting %s", hostname(b.host))
_instance_restart(b, admin, zk, healthcheck_retries, healthcheck_wait,
pre_stop)
def inits_from_iterable_of_Connections(self):
g = Group.from_connections((Connection("foo"), Connection("bar")))
assert len(g) == 2
assert g[1].host == "bar"
def inits_from_iterable_of_Connections(self):
g = Group.from_connections((Connection("foo"), Connection("bar")))
assert len(g) == 2
assert g[1].host == "bar"
def close_closes_all_member_connections(self):
cxns = [Mock(name=x) for x in ("foo", "bar", "biz")]
g = Group.from_connections(cxns)
g.close()
for c in cxns:
c.close.assert_called_once_with()
def inits_from_iterable_of_Connections(self):
g = Group.from_connections((Connection('foo'), Connection('bar')))
assert len(g) == 2
assert g[1].host == 'bar'
with c.local("cd redis-6.2.1"):
c.local("make clean")
c.local("tar -czf redis-3.2.8.tar.gz redis-3.2.8")
@task
def deploy(c):
c.put("redis-3.2.8.tar.gz", "/tmp/redis-3.2.8.tar.gz")
with c.cd("/tmp"):
c.run("tar xzf redis-3.2.8.tar.gz")
with c.cd("redis-3.2.8"):
c.sudo("make install")
@task
def clean_file(c):
with c.cd("/tmp"):
c.sudo("rm -rf redis-3.2.8.tar.gz")
c.sudo("rm -rf redis-3.2.8")
@task
def clean_local_file(c):
c.local("rm -rf redis-3.2.8.tar.gz")
@task
def install(c):
test(c)
for c in Group('10.8.100.3'):
install(c)
def connect(hosts):
group = Group(*hosts)
return group
def not_implemented_in_base_class(self):
Group().run()
def simple_command(self):
group = Group("localhost", "127.0.0.1")
result = group.run("echo foo", hide=True)
outs = [x.stdout.strip() for x in result.values()]
assert ["foo", "foo"] == outs
def may_be_empty(self):
assert len(Group()) == 0
def takes_splat_arg_of_host_strings(self):
g = Group("foo", "bar")
assert g[0].host == "foo"
assert g[1].host == "bar"