def plot(cls, params_dict):
try:
params = PlotParameters(params_dict)
except PlotError as e:
raise PlotError('Invalid nodes or bench parameters', e)
# Aggregate the logs.
LogAggregator(params.max_latency).print()
# Load the aggregated log files.
robustness_files, latency_files, tps_files = [], [], []
tx_size = params.tx_size
for f in params.faults:
for n in params.nodes:
robustness_files += glob(
PathMaker.agg_file('robustness', n, 'x', tx_size, f, 'any')
)
latency_files += glob(
PathMaker.agg_file('latency', n, 'any', tx_size, f, 'any')
)
for l in params.max_latency:
tps_files += glob(
PathMaker.agg_file('tps', 'x', 'any', tx_size, f, l)
)
# Make the plots.
cls.plot_robustness(robustness_files)
cls.plot_latency(latency_files)
cls.plot_tps(tps_files)
def print(self):
if not os.path.exists(PathMaker.plots_path()):
os.makedirs(PathMaker.plots_path())
results = [
self._print_latency(),
self._print_tps(),
self._print_robustness()
]
for records in results:
for setup, values in records.items():
data = '\n'.join(f' Variable value: X={x}\n{y}'
for x, y in values)
string = ('\n'
'-----------------------------------------\n'
' RESULTS:\n'
'-----------------------------------------\n'
f'{setup}'
'\n'
f'{data}'
'-----------------------------------------\n')
filename = PathMaker.agg_file(setup.nodes,
setup.rate,
setup.tx_size,
max_latency=setup.max_latency)
with open(filename, 'w') as f:
f.write(string)
def _run_single(self,
hosts,
rate,
bench_parameters,
node_parameters,
debug=False):
Print.info('Booting testbed...')
# Kill any potentially unfinished run and delete logs.
self.kill(hosts=hosts, delete_logs=True)
Print.info('Killed previous instances')
sleep(10)
# Run the clients (they will wait for the nodes to be ready).
# Filter all faulty nodes from the client addresses (or they will wait
# for the faulty nodes to be online).
committee = Committee.load(PathMaker.committee_file())
addresses = [f'{x}:{self.settings.front_port}' for x in hosts]
rate_share = ceil(rate / committee.size()) # Take faults into account.
timeout = node_parameters.timeout_delay
client_logs = [PathMaker.client_log_file(i) for i in range(len(hosts))]
for host, addr, log_file in zip(hosts, addresses, client_logs):
cmd = CommandMaker.run_client(addr,
bench_parameters.tx_size,
rate_share,
timeout,
nodes=addresses)
self._background_run(host, cmd, log_file)
Print.info('Clients boosted...')
sleep(10)
# Run the nodes.
key_files = [PathMaker.key_file(i) for i in range(len(hosts))]
dbs = [PathMaker.db_path(i) for i in range(len(hosts))]
node_logs = [PathMaker.node_log_file(i) for i in range(len(hosts))]
threshold_key_files = [
PathMaker.threshold_key_file(i) for i in range(len(hosts))
]
for host, key_file, threshold_key_file, db, log_file in zip(
hosts, key_files, threshold_key_files, dbs, node_logs):
cmd = CommandMaker.run_node(key_file,
threshold_key_file,
PathMaker.committee_file(),
db,
PathMaker.parameters_file(),
debug=debug)
self._background_run(host, cmd, log_file)
# Wait for the nodes to synchronize
Print.info('Waiting for the nodes to synchronize...')
sleep(node_parameters.timeout_delay / 1000)
# Wait for all transactions to be processed.
duration = bench_parameters.duration
for _ in progress_bar(range(100),
prefix=f'Running benchmark ({duration} sec):'):
sleep(ceil(duration / 100))
self.kill(hosts=hosts, delete_logs=False)
def _plot(self, x_label, y_label, y_axis, z_axis, type):
plt.figure()
markers = cycle(['o', 'v', 's', 'p', 'D', 'P'])
self.results.sort(key=self._natural_keys, reverse=(type == 'tps'))
for result in self.results:
y_values, y_err = y_axis(result)
x_values = self._variable(result)
if len(y_values) != len(y_err) or len(y_err) != len(x_values):
raise PlotError('Unequal number of x, y, and y_err values')
plt.errorbar(
x_values, y_values, yerr=y_err, label=z_axis(result),
linestyle='dotted', marker=next(markers), capsize=3
)
plt.legend(loc='lower center', bbox_to_anchor=(0.5, 1), ncol=2)
plt.xlim(xmin=0)
plt.ylim(bottom=0)
plt.xlabel(x_label)
plt.ylabel(y_label[0])
plt.grid()
ax = plt.gca()
ax.xaxis.set_major_formatter(StrMethodFormatter('{x:,.0f}'))
ax.yaxis.set_major_formatter(StrMethodFormatter('{x:,.0f}'))
if len(y_label) > 1:
secaxy = ax.secondary_yaxis(
'right', functions=(self._tps2bps, self._bps2tps)
)
secaxy.set_ylabel(y_label[1])
secaxy.yaxis.set_major_formatter(StrMethodFormatter('{x:,.0f}'))
for x in ['pdf', 'png']:
plt.savefig(PathMaker.plot_file(type, x), bbox_inches='tight')
def _plot(self, x_label, y_label, y_axis, z_axis, type):
plt.figure()
for result in self.results:
y_values, y_err = y_axis(result)
x_values = self._variable(result)
if len(y_values) != len(y_err) or len(y_err) != len(x_values):
raise PlotError('Unequal number of x, y, and y_err values')
plt.errorbar(
x_values, y_values, yerr=y_err, # uplims=True, lolims=True,
marker='o', label=z_axis(result), linestyle='dotted'
)
# if type == 'latency':
# plt.yscale('log')
plt.xlim(xmin=0)
plt.ylim(bottom=0)
plt.xlabel(x_label)
plt.ylabel(y_label[0])
plt.legend(loc='upper left')
ax = plt.gca()
ax.xaxis.set_major_formatter(StrMethodFormatter('{x:,.0f}'))
ax.yaxis.set_major_formatter(StrMethodFormatter('{x:,.0f}'))
if len(y_label) > 1:
secaxy = ax.secondary_yaxis(
'right', functions=(self._tps2bps, self._bps2tps)
)
secaxy.set_ylabel(y_label[1])
secaxy.yaxis.set_major_formatter(StrMethodFormatter('{x:,.0f}'))
for x in ['pdf', 'png']:
plt.savefig(PathMaker.plot_file(type, x), bbox_inches='tight')
def _logs(self, hosts):
# Delete local logs (if any).
cmd = CommandMaker.clean_logs()
subprocess.run([cmd], shell=True, stderr=subprocess.DEVNULL)
# Download log files.
progress = progress_bar(hosts, prefix='Downloading logs:')
for i, host in enumerate(progress):
c = Connection(host, user='******', connect_kwargs=self.connect)
c.get(PathMaker.node_log_file(i), local=PathMaker.node_log_file(i))
c.get(PathMaker.client_log_file(i),
local=PathMaker.client_log_file(i))
# Parse logs and return the parser.
Print.info('Parsing logs and computing performance...')
return LogParser.process(PathMaker.logs_path())
def plot_tps(cls, z_axis):
assert hasattr(z_axis, '__call__')
x_label = 'Committee size'
y_label = ['Throughput (tx/s)', 'Throughput (MB/s)']
files = glob(PathMaker.agg_file('x', 'any', r'*', r'*'))
ploter = cls(files)
ploter._plot(x_label, y_label, ploter._tps, z_axis, 'tps')
def plot_latency(cls, z_axis):
assert hasattr(z_axis, '__call__')
x_label = 'Throughput (tx/s)'
y_label = ['Latency (ms)']
files = glob(PathMaker.agg_file(r'[0-9]*', 'any', r'*', 'any'))
ploter = cls(files)
ploter._plot(x_label, y_label, ploter._latency, z_axis, 'latency')
def plot_robustness(cls, z_axis):
assert hasattr(z_axis, '__call__')
x_label = 'Input rate (tx/s)'
y_label = ['Throughput (tx/s)', 'Throughput (MB/s)']
files = glob(PathMaker.agg_file(r'[0-9]*', 'x', r'*', 'any'))
ploter = cls(files)
ploter._plot(x_label, y_label, ploter._tps, z_axis, 'robustness')
def __init__(self):
data = ''
for filename in glob(join(PathMaker.results_path(), '*.txt')):
with open(filename, 'r') as f:
data += f.read()
records = defaultdict(list)
for chunk in data.replace(',', '').split('SUMMARY')[1:]:
if chunk:
records[Setup.from_str(chunk)] += [Result.from_str(chunk)]
self.records = {k: Result.aggregate(v) for k, v in records.items()}
def run(self, bench_parameters_dict, node_parameters_dict, debug=False):
assert isinstance(debug, bool)
Print.heading('Starting remote benchmark')
try:
bench_parameters = BenchParameters(bench_parameters_dict)
node_parameters = NodeParameters(node_parameters_dict)
except ConfigError as e:
raise BenchError('Invalid nodes or bench parameters', e)
# Select which hosts to use.
selected_hosts = self._select_hosts(bench_parameters)
if not selected_hosts:
Print.warn('There are not enough instances available')
return
# Update nodes.
try:
self._update(selected_hosts)
except (GroupException, ExecutionError) as e:
e = FabricError(e) if isinstance(e, GroupException) else e
raise BenchError('Failed to update nodes', e)
# Run benchmarks.
for n in bench_parameters.nodes:
for r in bench_parameters.rate:
Print.heading(f'\nRunning {n} nodes (input rate: {r:,} tx/s)')
hosts = selected_hosts[:n]
# Upload all configuration files.
try:
self._config(hosts, node_parameters)
except (subprocess.SubprocessError, GroupException) as e:
e = FabricError(e) if isinstance(e, GroupException) else e
Print.error(BenchError('Failed to configure nodes', e))
continue
# Run the benchmark.
for i in range(bench_parameters.runs):
Print.heading(f'Run {i+1}/{bench_parameters.runs}')
try:
self._run_single(hosts, r, bench_parameters,
node_parameters, debug)
self._logs(hosts).print(
PathMaker.result_file(n, r,
bench_parameters.tx_size))
except (subprocess.SubprocessError, GroupException,
ParseError) as e:
self.kill(hosts=hosts)
if isinstance(e, GroupException):
e = FabricError(e)
Print.error(BenchError('Benchmark failed', e))
continue
def _config(self, hosts, node_parameters):
Print.info('Generating configuration files...')
# Cleanup all local configuration files.
cmd = CommandMaker.cleanup()
subprocess.run([cmd], shell=True, stderr=subprocess.DEVNULL)
# Recompile the latest code.
cmd = CommandMaker.compile().split()
subprocess.run(cmd, check=True, cwd=PathMaker.node_crate_path())
# Create alias for the client and nodes binary.
cmd = CommandMaker.alias_binaries(PathMaker.binary_path())
subprocess.run([cmd], shell=True)
# Generate configuration files.
keys = []
key_files = [PathMaker.key_file(i) for i in range(len(hosts))]
for filename in key_files:
cmd = CommandMaker.generate_key(filename).split()
subprocess.run(cmd, check=True)
keys += [Key.from_file(filename)]
names = [x.name for x in keys]
consensus_addr = [f'{x}:{self.settings.consensus_port}' for x in hosts]
mempool_addr = [f'{x}:{self.settings.mempool_port}' for x in hosts]
front_addr = [f'{x}:{self.settings.front_port}' for x in hosts]
committee = Committee(names, consensus_addr, mempool_addr, front_addr)
committee.print(PathMaker.committee_file())
node_parameters.print(PathMaker.parameters_file())
# Cleanup all nodes.
cmd = f'{CommandMaker.cleanup()} || true'
g = Group(*hosts, user='******', connect_kwargs=self.connect)
g.run(cmd, hide=True)
# Upload configuration files.
progress = progress_bar(hosts, prefix='Uploading config files:')
for i, host in enumerate(progress):
c = Connection(host, user='******', connect_kwargs=self.connect)
c.put(PathMaker.committee_file(), '.')
c.put(PathMaker.key_file(i), '.')
c.put(PathMaker.parameters_file(), '.')
return committee
def __init__(self, max_latencies):
assert isinstance(max_latencies, list)
assert all(isinstance(x, int) for x in max_latencies)
self.max_latencies = max_latencies
data = ''
for filename in glob(join(PathMaker.results_path(), '*.txt')):
with open(filename, 'r') as f:
data += f.read()
records = defaultdict(list)
for chunk in data.replace(',', '').split('SUMMARY')[1:]:
if chunk:
records[Setup.from_str(chunk)] += [Result.from_str(chunk)]
self.records = {k: Result.aggregate(v) for k, v in records.items()}
def _run_single(self,
hosts,
rate,
bench_parameters,
node_parameters,
debug=False):
Print.info('Booting testbed...')
# Kill any potentially unfinished run and delete logs.
self.kill(hosts=hosts, delete_logs=True)
# Run the clients (they will wait for the nodes to be ready).
committee = Committee.load(PathMaker.committee_file())
addresses = committee.front_addresses()
rate_share = ceil(rate / committee.size())
timeout = node_parameters.timeout_delay
client_logs = [PathMaker.client_log_file(i) for i in range(len(hosts))]
for host, addr, log_file in zip(hosts, addresses, client_logs):
cmd = CommandMaker.run_client(addr,
bench_parameters.tx_size,
rate_share,
timeout,
nodes=addresses)
self._background_run(host, cmd, log_file)
# Run the nodes.
key_files = [PathMaker.key_file(i) for i in range(len(hosts))]
dbs = [PathMaker.db_path(i) for i in range(len(hosts))]
node_logs = [PathMaker.node_log_file(i) for i in range(len(hosts))]
for host, key_file, db, log_file in zip(hosts, key_files, dbs,
node_logs):
cmd = CommandMaker.run_node(key_file,
PathMaker.committee_file(),
db,
PathMaker.parameters_file(),
debug=debug)
self._background_run(host, cmd, log_file)
# Wait for the nodes to synchronize
Print.info('Waiting for the nodes to synchronize...')
sleep(2 * node_parameters.timeout_delay / 1000)
# Wait for all transactions to be processed.
duration = bench_parameters.duration
for _ in progress_bar(range(20),
prefix=f'Running benchmark ({duration} sec):'):
sleep(ceil(duration / 20))
self.kill(hosts=hosts, delete_logs=False)
def run(self, debug=False):
assert isinstance(debug, bool)
Print.heading('Starting local benchmark')
# Kill any previous testbed.
self._kill_nodes()
try:
Print.info('Setting up testbed...')
nodes, rate = self.nodes[0], self.rate[0]
# Cleanup all files.
cmd = f'{CommandMaker.clean_logs()} ; {CommandMaker.cleanup()}'
subprocess.run([cmd], shell=True, stderr=subprocess.DEVNULL)
sleep(0.5) # Removing the store may take time.
# Recompile the latest code.
cmd = CommandMaker.compile().split()
subprocess.run(cmd, check=True, cwd=PathMaker.node_crate_path())
# Create alias for the client and nodes binary.
cmd = CommandMaker.alias_binaries(PathMaker.binary_path())
subprocess.run([cmd], shell=True)
# Generate configuration files.
keys = []
key_files = [PathMaker.key_file(i) for i in range(nodes)]
for filename in key_files:
cmd = CommandMaker.generate_key(filename).split()
subprocess.run(cmd, check=True)
keys += [Key.from_file(filename)]
names = [x.name for x in keys]
committee = LocalCommittee(names, self.BASE_PORT)
committee.print(PathMaker.committee_file())
self.node_parameters.print(PathMaker.parameters_file())
# Run the clients (they will wait for the nodes to be ready).
addresses = committee.front_addresses()
rate_share = ceil(rate / nodes)
timeout = self.node_parameters.timeout_delay
client_logs = [PathMaker.client_log_file(i) for i in range(nodes)]
for addr, log_file in zip(addresses, client_logs):
cmd = CommandMaker.run_client(addr, self.tx_size, rate_share,
timeout)
self._background_run(cmd, log_file)
# Run the nodes.
dbs = [PathMaker.db_path(i) for i in range(nodes)]
node_logs = [PathMaker.node_log_file(i) for i in range(nodes)]
for key_file, db, log_file in zip(key_files, dbs, node_logs):
cmd = CommandMaker.run_node(key_file,
PathMaker.committee_file(),
db,
PathMaker.parameters_file(),
debug=debug)
self._background_run(cmd, log_file)
# Wait for the nodes to synchronize
Print.info('Waiting for the nodes to synchronize...')
sleep(2 * self.node_parameters.timeout_delay / 1000)
# Wait for all transactions to be processed.
Print.info(f'Running benchmark ({self.duration} sec)...')
sleep(self.duration)
self._kill_nodes()
# Parse logs and return the parser.
Print.info('Parsing logs...')
return LogParser.process('./logs')
except (subprocess.SubprocessError, ParseError) as e:
self._kill_nodes()
raise BenchError('Failed to run benchmark', e)
def run(self, bench_parameters_dict, node_parameters_dict, debug=False):
assert isinstance(debug, bool)
Print.heading('Starting remote benchmark')
try:
bench_parameters = BenchParameters(bench_parameters_dict)
node_parameters = NodeParameters(node_parameters_dict)
except ConfigError as e:
raise BenchError('Invalid nodes or bench parameters', e)
# Select which hosts to use.
selected_hosts = self._select_hosts(bench_parameters)
if not selected_hosts:
Print.warn('There are not enough instances available')
return
# Update nodes.
try:
self._update(selected_hosts)
except (GroupException, ExecutionError) as e:
e = FabricError(e) if isinstance(e, GroupException) else e
raise BenchError('Failed to update nodes', e)
if node_parameters.protocol == 0:
Print.info('Running HotStuff')
elif node_parameters.protocol == 1:
Print.info('Running AsyncHotStuff')
elif node_parameters.protocol == 2:
Print.info('Running TwoChainVABA')
else:
Print.info('Wrong protocol type!')
return
Print.info(f'{bench_parameters.faults} faults')
Print.info(
f'Timeout {node_parameters.timeout_delay} ms, Network delay {node_parameters.network_delay} ms'
)
Print.info(f'DDOS attack {node_parameters.ddos}')
hosts = selected_hosts[:bench_parameters.nodes[0]]
# Upload all configuration files.
try:
self._config(hosts, node_parameters)
except (subprocess.SubprocessError, GroupException) as e:
e = FabricError(e) if isinstance(e, GroupException) else e
Print.error(BenchError('Failed to configure nodes', e))
# Run benchmarks.
for n in bench_parameters.nodes:
for r in bench_parameters.rate:
Print.heading(f'\nRunning {n} nodes (input rate: {r:,} tx/s)')
hosts = selected_hosts[:n]
# # Upload all configuration files.
# try:
# self._config(hosts, node_parameters)
# except (subprocess.SubprocessError, GroupException) as e:
# e = FabricError(e) if isinstance(e, GroupException) else e
# Print.error(BenchError('Failed to configure nodes', e))
# continue
# Do not boot faulty nodes.
faults = bench_parameters.faults
hosts = hosts[:n - faults]
protocol = node_parameters.protocol
ddos = node_parameters.ddos
# Run the benchmark.
for i in range(bench_parameters.runs):
Print.heading(f'Run {i+1}/{bench_parameters.runs}')
try:
self._run_single(hosts, r, bench_parameters,
node_parameters, debug)
self._logs(hosts, faults, protocol, ddos).print(
PathMaker.result_file(n, r,
bench_parameters.tx_size,
faults))
except (subprocess.SubprocessError, GroupException,
ParseError) as e:
self.kill(hosts=hosts)
if isinstance(e, GroupException):
e = FabricError(e)
Print.error(BenchError('Benchmark failed', e))
continue
def run(self, debug=False):
assert isinstance(debug, bool)
Print.heading('Starting local benchmark')
# Kill any previous testbed.
self._kill_nodes()
try:
Print.info('Setting up testbed...')
nodes, rate = self.nodes[0], self.rate[0]
# Cleanup all files.
cmd = f'{CommandMaker.clean_logs()} ; {CommandMaker.cleanup()}'
subprocess.run([cmd], shell=True, stderr=subprocess.DEVNULL)
sleep(0.5) # Removing the store may take time.
# Recompile the latest code.
cmd = CommandMaker.compile().split()
subprocess.run(cmd, check=True, cwd=PathMaker.node_crate_path())
# Create alias for the client and nodes binary.
cmd = CommandMaker.alias_binaries(PathMaker.binary_path())
subprocess.run([cmd], shell=True)
# Generate configuration files.
keys = []
key_files = [PathMaker.key_file(i) for i in range(nodes)]
for filename in key_files:
cmd = CommandMaker.generate_key(filename).split()
subprocess.run(cmd, check=True)
keys += [Key.from_file(filename)]
# Generate threshold signature files.
cmd = './node threshold_keys'
for i in range(nodes):
cmd += ' --filename ' + PathMaker.threshold_key_file(i)
# print(cmd)
cmd = cmd.split()
subprocess.run(cmd, capture_output=True, check=True)
names = [x.name for x in keys]
tss_keys = []
for i in range(nodes):
tss_keys += [TSSKey.from_file(PathMaker.threshold_key_file(i))]
ids = [x.id for x in tss_keys]
committee = LocalCommittee(names, ids, self.BASE_PORT)
committee.print(PathMaker.committee_file())
self.node_parameters.print(PathMaker.parameters_file())
# Do not boot faulty nodes.
nodes = nodes - self.faults
# Run the clients (they will wait for the nodes to be ready).
addresses = committee.front
rate_share = ceil(rate / nodes)
timeout = self.node_parameters.timeout_delay
client_logs = [PathMaker.client_log_file(i) for i in range(nodes)]
for addr, log_file in zip(addresses, client_logs):
cmd = CommandMaker.run_client(addr, self.tx_size, rate_share,
timeout)
self._background_run(cmd, log_file)
if self.node_parameters.protocol == 0:
Print.info('Running HotStuff')
elif self.node_parameters.protocol == 1:
Print.info('Running Async HotStuff')
elif self.node_parameters.protocol == 2:
Print.info('Running TwoChainVABA')
else:
Print.info('Wrong protocol type!')
return
Print.info(f'{self.faults} faults')
Print.info(
f'Timeout {self.node_parameters.timeout_delay} ms, Network delay {self.node_parameters.network_delay} ms'
)
Print.info(f'DDOS attack {self.node_parameters.ddos}')
# Run the nodes.
dbs = [PathMaker.db_path(i) for i in range(nodes)]
node_logs = [PathMaker.node_log_file(i) for i in range(nodes)]
threshold_key_files = [
PathMaker.threshold_key_file(i) for i in range(nodes)
]
for key_file, threshold_key_file, db, log_file in zip(
key_files, threshold_key_files, dbs, node_logs):
cmd = CommandMaker.run_node(key_file,
threshold_key_file,
PathMaker.committee_file(),
db,
PathMaker.parameters_file(),
debug=debug)
self._background_run(cmd, log_file)
# Wait for the nodes to synchronize
Print.info('Waiting for the nodes to synchronize...')
sleep(2 * self.node_parameters.timeout_delay / 1000)
# Wait for all transactions to be processed.
Print.info(f'Running benchmark ({self.duration} sec)...')
sleep(self.duration)
self._kill_nodes()
# Parse logs and return the parser.
Print.info('Parsing logs...')
return LogParser.process('./logs', self.faults,
self.node_parameters.protocol,
self.node_parameters.ddos)
except (subprocess.SubprocessError, ParseError) as e:
self._kill_nodes()
raise BenchError('Failed to run benchmark', e)