def _drop_packets_between(self, src_node_id, dst_node_id, drop_rate_percentage=100):
""" Drop all packets send between src_node_id and dst_node_id with probability drop_rate_percentage """
if self.comm_type == bft_config.COMM_TYPE_TCP_TLS:
# Get source and destination ports from already initialized self.connections
(src_port, dst_port) = self.connections[(src_node_id, dst_node_id)]
else:
# Source and destination ports are well-known ports
src_port = bft_config.bft_msg_port_from_node_id(src_node_id)
dst_port = bft_config.bft_msg_port_from_node_id(dst_node_id)
pass
self._drop_packets_between_ports(src_port, dst_port, drop_rate_percentage)
def setUpClass(cls):
cls.origdir = os.getcwd()
cls.testdir = tempfile.mkdtemp()
cls.builddir = os.path.abspath("../../build")
cls.toolsdir = os.path.join(cls.builddir, "tools")
cls.serverbin = os.path.join(cls.builddir,"tests/simpleTest/server")
os.chdir(cls.testdir)
cls.generateKeys()
cls.config = bft_config.Config(4, 1, 0, 4096, 1000, 50, "")
cls.replicas = [bft_config.Replica(id=i,
ip="127.0.0.1",
port=bft_config.bft_msg_port_from_node_id(i),
metrics_port=bft_config.metrics_port_from_node_id(i))
for i in range(0,4)]
print("Running tests in {}".format(cls.testdir))
cls.startServers()
def __init__(self, config, replicas, background_nursery, ro_replicas=[]):
super().__init__(config, replicas, ro_replicas)
self.sock = trio.socket.socket(trio.socket.AF_INET,
trio.socket.SOCK_DGRAM)
self.port = bft_msg_port_from_node_id(self.client_id)
def _build_port_connectivity_matrix(self, time_to_wait_sec=5.0, time_between_retries=0.25):
"""
Used only in TCP communication.
Build a connectivity matrix self.connections, a dictionary which maps a pair src_node_id, dest_node_id
into a matching source_port and dest_port.
Having this data structure allows to translate "block communication between node id X and node it Y" very easily
onto the underlaying ports.
Since module cannot know if all replicas started or no, it retries the discovery in a loop, with
time_between_retries seconds between retries. It fails after time_to_wait_sec seconds (This should never happen)
"""
assert isinstance(time_to_wait_sec, float) and isinstance(time_between_retries, float)
wait_until_time = time.time() + time_to_wait_sec
server_msg_ports = [bft_config.bft_msg_port_from_node_id(i) for i in self.bft_network.all_replicas()]
all_connections_established = False
num_replicas = self.bft_network.config.n
# Rules to calculate num_expected_connections:
# 1) Each client/replica connect to all other replicas with lower ID. Hence, replica 0 does not connect to
# anyone.
# 2) Each active client connects to all other replicas.
# 3) Clients do not connect to each other.
# 4) Each connection is bi-directional (hence 2 lines in output.
# 5) Each connection is uni-directional for our purpose.
# So between replicas we get 2 * (0 + 1 + 2... + N-1) = N * (N -1) connections, where N is the number of
# replicas and for the clients <-> replicas we get 2 * N * num_clients connections
num_expected_connections = num_replicas * (num_replicas - 1) + (len(self.all_client_ids) * num_replicas * 2)
while time.time() < wait_until_time:
try:
# Use 'netstat -tpun' and grep on server known ports, to get all established TCP connection details
netstat_output = subprocess.run(['netstat', '-tpun'],
check=True, universal_newlines=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# stringify the grep command regex
grep_format = "127.0.0.1:%d .* ESTABLISHED "
grep_str = ""
grep_reg_or = "\\|"
for i in range(len(server_msg_ports)):
grep_str += (grep_format % server_msg_ports[i]) + grep_reg_or
grep_str = grep_str[:len(grep_str) - len(grep_reg_or)]
grep_output = subprocess.run(['grep', grep_str],
check=True, universal_newlines=True, input=netstat_output.stdout,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
netstat_lines = grep_output.stdout.splitlines()
if len(netstat_lines) == num_expected_connections:
# Success, all expected data is parsed, break.
all_connections_established = True
break
else:
# Failure, connections are not yet fully established, need to retry
time.sleep(time_between_retries)
continue
except subprocess.CalledProcessError:
# Failure, connections are not yet fully established, need to retry
time.sleep(time_between_retries)
continue
assert all_connections_established, "Failed to build port connectivity matrix using netstat!"
# Here we pars the data we got from grep_output
for line in netstat_lines:
tokens = list(filter(None, split(' |:|/', line)))
src_port = int(tokens[4])
dst_port = int(tokens[6])
src_proc_id = int(tokens[8])
assert (src_port in server_msg_ports) ^ (dst_port in server_msg_ports)
src_node_id = self._port_to_node_id(src_port, dst_port, src_proc_id)
dst_node_id = self._port_to_node_id(dst_port, src_port)
self.connections[(src_node_id, dst_node_id)] = (src_port, dst_port)
assert len(self.connections) == num_expected_connections