def start_and_wait_for_server(nursery: Nursery, port_generator: Iterator[Port],
node: NodeConfig,
retry_timeout: int) -> Optional[RunningNode]:
"""Start the Raiden node and waits for the REST API to be available,
returns None if the script is being shutdown.
"""
# redirect the process output for debugging
os.makedirs(os.path.expanduser(node.data_dir), exist_ok=True)
stdout = open(os.path.join(node.data_dir, "stress_test.out"), "a")
port = next(port_generator)
api_url = f"{node.interface}:{port}"
running_url = URL(f"http://{api_url}")
process_args = node.args + ["--api-address", api_url]
process = nursery.exec_under_watch(process_args,
bufsize=UNBUFERRED,
stdout=stdout,
stderr=STDOUT)
if process is not None:
wait_for_status_ready(running_url, retry_timeout)
return RunningNode(process, node, running_url)
return None
def run(config: Config, nursery: Nursery) -> None:
for i, qty_of_rooms in enumerate(
batch_size(config.target_qty_of_chat_rooms,
config.qty_of_new_rooms_per_iteration)):
log_file = os.path.join(config.logdir, str(i))
script_args: List[str] = [
GENERATE_MESSAGES_SCRIPT,
"--concurrent-messages",
str(config.concurrent_messages_per_room),
"--chat-rooms",
str(qty_of_rooms),
log_file,
config.sender_matrix_server_url,
config.receiver_matrix_server_url,
]
nursery.exec_under_watch(script_args)
time.sleep(config.wait_before_next_iteration)
def run_profiler(nursery: Nursery, running_nodes: List[RunningNode],
profiler_data_directory: str) -> List[Popen]:
os.makedirs(os.path.expanduser(profiler_data_directory), exist_ok=True)
profiler_processes: List[Popen] = list()
for node in running_nodes:
args = [
"py-spy",
"record",
"--pid",
str(node.process.pid),
"--output",
os.path.join(
profiler_data_directory,
f"{node.config.address}-{datetime.utcnow().isoformat()}.data",
),
]
profiler = Popen(args, stdout=DEVNULL, stderr=DEVNULL)
nursery.exec_under_watch(profiler)
return profiler_processes
def start_and_wait_for_all_servers(
nursery: Nursery,
port_generator: Iterator[Port],
nodes_config: List[NodeConfig],
retry_timeout: int,
) -> Optional[List[RunningNode]]:
"""Starts all nodes under the nursery, returns a list of `RunningNode`s or
None if the script is shuting down.
Important Note:
`None` is not always returned if the script is shutting down! Due to race
conditions it is possible for all processes to be spawned, and only
afterwards the nursery is closed. IOW: At this stage `None` will only be
returned if spawning the process fails (e.g. the binary name is wrong),
however, if the subprocess is spawned and runs for some time, and *then*
crashes, `None` will **not** be returned here (e.g. if the ethereum node is
not available). For the second case, the `stop_event` will be set.
Because of the above, for proper error handling, checking only the return
value is **not** sufficient. The most reliable approach is to execute new
logic in greenlets spawned with `spawn_under_watch` and let errors fall
through.
"""
greenlets = set(
nursery.spawn_under_watch(start_and_wait_for_server, nursery,
port_generator, node, retry_timeout)
for node in nodes_config)
all_running_nodes = []
for g in gevent.joinall(greenlets, raise_error=True):
running_node = g.get()
if running_node is None:
return None
all_running_nodes.append(running_node)
return all_running_nodes
def restart_network(
nursery: Nursery,
port_generator: Iterator[Port],
running_nodes: List[RunningNode],
retry_timeout: int,
) -> Optional[List[RunningNode]]:
"""Stop all `RunningNode`s and start them again under the nursery, returns
None if the script is shuting down.
"""
greenlets = set(
nursery.spawn_under_watch(restart_and_wait_for_server, nursery,
port_generator, node, retry_timeout)
for node in running_nodes)
all_running_nodes = []
for g in gevent.joinall(greenlets, raise_error=True):
running_node = g.get()
if running_node is None:
return None
all_running_nodes.append(running_node)
return all_running_nodes
def start_and_wait_for_all_servers(
nursery: Nursery,
port_generator: Iterator[Port],
nodes_config: List[NodeConfig],
retry_timeout: int,
) -> Optional[List[RunningNode]]:
"""Starts all nodes under the nursery, returns a list of `RunningNode`s or
None if the script is shuting down.
"""
greenlets = set(
nursery.spawn_under_watch(start_and_wait_for_server, nursery,
port_generator, node, retry_timeout)
for node in nodes_config)
all_running_nodes = []
for g in gevent.joinall(greenlets, raise_error=True):
running_node = g.get()
if running_node is None:
return None
all_running_nodes.append(running_node)
return all_running_nodes