def start_cluster(cmd: str, args: List[str]) -> clusterlib.ClusterLib:
"""Start cluster."""
args_str = " ".join(args)
args_str = f" {args_str}" if args_str else ""
LOGGER.info(f"Starting cluster with `{cmd}{args_str}`.")
helpers.run_command(f"{cmd}{args_str}", workdir=get_cluster_env().work_dir)
LOGGER.info("Cluster started.")
return get_cluster_type().get_cluster_obj()
def restart_all_nodes(instance_num: Optional[int] = None) -> None:
"""Restart all Cardano nodes of the running cluster."""
LOGGER.info("Restarting all cluster nodes.")
if instance_num is None:
instance_num = get_cluster_env().instance_num
supervisor_port = get_cluster_type().cluster_scripts.get_instance_ports(instance_num).supervisor
try:
helpers.run_command(f"supervisorctl -s http://localhost:{supervisor_port} restart nodes:")
except Exception as exc:
LOGGER.debug(f"Failed to restart cluster nodes: {exc}")
def restart_all_nodes() -> None:
"""Restart all Cardano nodes of the running cluster."""
LOGGER.info("Restarting all cluster nodes.")
cluster_env = get_cluster_env()
supervisor_port = (get_cluster_type().cluster_scripts.get_instance_ports(
cluster_env.instance_num).supervisor)
try:
helpers.run_command(
f"supervisorctl -s http://localhost:{supervisor_port} restart nodes:",
workdir=cluster_env.work_dir,
)
except Exception as exc:
LOGGER.debug(f"Failed to restart cluster nodes: {exc}")
def restart_node(node_name: str) -> None:
"""Restart single node of the running cluster."""
LOGGER.info(f"Restarting cluster node `{node_name}`.")
cluster_env = get_cluster_env()
supervisor_port = (get_cluster_type().cluster_scripts.get_instance_ports(
cluster_env.instance_num).supervisor)
try:
helpers.run_command(
f"supervisorctl -s http://localhost:{supervisor_port} restart {node_name}",
workdir=cluster_env.work_dir,
)
except Exception as exc:
LOGGER.debug(f"Failed to restart cluster node `{node_name}`: {exc}")
def cli_has(command: str) -> bool:
"""Check if a cardano-cli subcommand or argument is available.
E.g. `cli_has("query leadership-schedule --next")`
"""
err_str = ""
try:
helpers.run_command(f"cardano-cli {command}")
except AssertionError as err:
err_str = str(err)
else:
return True
cmd_err = err_str.split(":", maxsplit=1)[1].strip()
return not cmd_err.startswith("Invalid")
def restart_nodes(node_names: List[str]) -> None:
"""Restart list of Cardano nodes of the running cluster."""
LOGGER.info(f"Restarting cluster nodes {node_names}.")
cluster_env = get_cluster_env()
supervisor_port = (get_cluster_type().cluster_scripts.get_instance_ports(
cluster_env.instance_num).supervisor)
for node_name in node_names:
try:
helpers.run_command(
f"supervisorctl -s http://localhost:{supervisor_port} restart nodes:{node_name}",
workdir=cluster_env.work_dir,
)
except Exception as exc:
LOGGER.debug(
f"Failed to restart cluster node `{node_name}`: {exc}")
def stop_all_clusters(self) -> None:
"""Stop all cluster instances."""
self._log("called `stop_all_clusters`")
# don't stop cluster if it was started outside of test framework
if configuration.DEV_CLUSTER_RUNNING:
LOGGER.warning(
"Ignoring request to stop clusters as 'DEV_CLUSTER_RUNNING' is set."
)
return
work_dir = cluster_nodes.get_cluster_env().work_dir
for instance_num in range(self.num_of_instances):
instance_dir = self.pytest_tmp_dir / f"{CLUSTER_DIR_TEMPLATE}{instance_num}"
if (not (instance_dir / CLUSTER_RUNNING_FILE).exists()
or (instance_dir / CLUSTER_STOPPED_FILE).exists()):
self._log(f"c{instance_num}: cluster instance not running")
continue
state_dir = work_dir / f"{cluster_nodes.STATE_CLUSTER}{instance_num}"
stop_script = state_dir / cluster_scripts.STOP_SCRIPT
if not stop_script.exists():
self._log(f"c{instance_num}: stop script doesn't exist!")
continue
self._log(
f"c{instance_num}: stopping cluster instance with `{stop_script}`"
)
try:
helpers.run_command(str(stop_script))
except Exception as err:
self._log(f"c{instance_num}: failed to stop cluster:\n{err}")
artifacts.save_start_script_coverage(
log_file=state_dir / CLUSTER_START_CMDS_LOG,
pytest_config=self.pytest_config,
)
artifacts.save_cluster_artifacts(save_dir=self.pytest_tmp_dir,
state_dir=state_dir)
shutil.rmtree(state_dir, ignore_errors=True)
helpers.touch(instance_dir / CLUSTER_STOPPED_FILE)
self._log(f"c{instance_num}: stopped cluster instance")
def services_action(
service_names: List[str], action: str, instance_num: Optional[int] = None
) -> None:
"""Restart list of services running on the running cluster."""
LOGGER.info(f"Performing '{action}' action on services {service_names}.")
if instance_num is None:
instance_num = get_cluster_env().instance_num
supervisor_port = get_cluster_type().cluster_scripts.get_instance_ports(instance_num).supervisor
for service_name in service_names:
try:
helpers.run_command(
f"supervisorctl -s http://localhost:{supervisor_port} {action} {service_name}"
)
except Exception as exc:
LOGGER.debug(f"Failed to restart service `{service_name}`: {exc}")
def create_script_context(cluster_obj: clusterlib.ClusterLib,
redeemer_file: Path,
tx_file: Optional[Path] = None) -> None:
"""Run the `create-script-context` command (available in plutus-examples)."""
if tx_file:
cmd_args = [
"create-script-context",
"--generate-tx",
str(tx_file),
"--out-file",
str(redeemer_file),
f"--{cluster_obj.protocol}-mode",
*cluster_obj.magic_args,
]
else:
cmd_args = ["create-script-context", "--out-file", str(redeemer_file)]
helpers.run_command(cmd_args)
assert redeemer_file.exists()
def _kill_supervisor(instance_num: int) -> None:
"""Kill supervisor process."""
port = f":{cluster_instances.get_instance_ports(instance_num).supervisor}"
netstat = helpers.run_command("netstat -plnt").decode().splitlines()
for line in netstat:
if port not in line:
continue
line = line.replace(" ", " ").strip()
pid = line.split(" ")[-1].split("/")[0]
os.kill(int(pid), 15)
return
def get_cardano_version(self) -> dict:
"""Return version info for cardano-node."""
out = helpers.run_command("cardano-node --version").decode().strip()
env_info, git_info, *__ = out.splitlines()
node, platform, ghc, *__ = env_info.split(" - ")
version_db = {
"cardano-node": node.split(" ")[-1],
"platform": platform,
"ghc": ghc,
"git_rev": git_info.split(" ")[-1],
}
return version_db
def get_dbsync_version(self) -> dict:
"""Return version info for db-sync."""
out = helpers.run_command(f"{configuration.DBSYNC_BIN} --version").decode().strip()
env_info, git_info, *__ = out.splitlines()
dbsync, platform, ghc, *__ = env_info.split(" - ")
version_db = {
"version": dbsync.split(" ")[-1],
"platform": platform,
"ghc": ghc,
"git_rev": git_info.split(" ")[-1],
}
return version_db
def filtered_ledger_state(cluster_obj: clusterlib.ClusterLib, ) -> str:
"""Get filtered output of `query ledger-state`."""
cardano_cmd = " ".join([
"cardano-cli",
"query",
"ledger-state",
*cluster_obj.magic_args,
f"--{cluster_obj.protocol}-mode",
])
# get rid of a huge amount of data we don't have any use for
cmd = (
f"{cardano_cmd} | jq -n --stream -c "
"'fromstream(inputs|select((length == 2 and .[0][1] == \"esLState\")|not))'"
)
return helpers.run_command(cmd, shell=True).decode("utf-8").strip()
def services_status(
service_names: Optional[List[str]] = None, instance_num: Optional[int] = None
) -> List[ServiceStatus]:
"""Return status info for list of services running on the running cluster (all by default)."""
if instance_num is None:
instance_num = get_cluster_env().instance_num
supervisor_port = get_cluster_type().cluster_scripts.get_instance_ports(instance_num).supervisor
service_names_arg = " ".join(service_names) if service_names else "all"
status_out = (
helpers.run_command(
f"supervisorctl -s http://localhost:{supervisor_port} status {service_names_arg}",
ignore_fail=True,
)
.decode()
.strip()
.split("\n")
)
statuses = []
for status_line in status_out:
service_name, status, *running_status = status_line.split()
if running_status and running_status[0] == "pid":
_pid, pid, _uptime, uptime, *other = running_status
message = " ".join(other)
else:
pid, uptime = "", ""
message = " ".join(running_status)
statuses.append(
ServiceStatus(
name=service_name,
status=status,
pid=int(pid.rstrip(",")) if pid else None,
uptime=uptime or None,
message=message,
)
)
return statuses
def _restart(self,
start_cmd: str = "",
stop_cmd: str = "") -> bool: # noqa: C901
"""Restart cluster.
Not called under global lock!
"""
# pylint: disable=too-many-branches
cluster_running_file = self.cm.instance_dir / CLUSTER_RUNNING_FILE
# don't restart cluster if it was started outside of test framework
if configuration.DEV_CLUSTER_RUNNING:
self.cm._log(
f"c{self.cm.cluster_instance_num}: ignoring restart, dev cluster is running"
)
if cluster_running_file.exists():
LOGGER.warning(
"Ignoring requested cluster restart as 'DEV_CLUSTER_RUNNING' is set."
)
else:
helpers.touch(cluster_running_file)
return True
# fail if cluster restart is forbidden and it was already started
if configuration.FORBID_RESTART and cluster_running_file.exists():
raise RuntimeError(
"Cannot restart cluster when 'FORBID_RESTART' is set.")
self.cm._log(
f"c{self.cm.cluster_instance_num}: called `_restart`, start_cmd='{start_cmd}', "
f"stop_cmd='{stop_cmd}'")
startup_files = cluster_nodes.get_cluster_type(
).cluster_scripts.prepare_scripts_files(
destdir=self.cm._create_startup_files_dir(
self.cm.cluster_instance_num),
instance_num=self.cm.cluster_instance_num,
start_script=start_cmd,
stop_script=stop_cmd,
)
state_dir = cluster_nodes.get_cluster_env().state_dir
self.cm._log(
f"c{self.cm.cluster_instance_num}: in `_restart`, new files "
f"start_cmd='{startup_files.start_script}', "
f"stop_cmd='{startup_files.stop_script}'")
excp: Optional[Exception] = None
for i in range(2):
if i > 0:
self.cm._log(
f"c{self.cm.cluster_instance_num}: failed to start cluster:\n{excp}\nretrying"
)
time.sleep(0.2)
try:
LOGGER.info(
f"Stopping cluster with `{startup_files.stop_script}`.")
helpers.run_command(str(startup_files.stop_script))
except Exception as err:
self.cm._log(
f"c{self.cm.cluster_instance_num}: failed to stop cluster:\n{err}"
)
# save artifacts only when produced during this test run
if cluster_running_file.exists():
artifacts.save_start_script_coverage(
log_file=state_dir / CLUSTER_START_CMDS_LOG,
pytest_config=self.cm.pytest_config,
)
artifacts.save_cluster_artifacts(
save_dir=self.cm.pytest_tmp_dir, state_dir=state_dir)
shutil.rmtree(state_dir, ignore_errors=True)
with contextlib.suppress(Exception):
_kill_supervisor(self.cm.cluster_instance_num)
try:
cluster_obj = cluster_nodes.start_cluster(
cmd=str(startup_files.start_script),
args=startup_files.start_script_args)
except Exception as err:
LOGGER.error(f"Failed to start cluster: {err}")
excp = err
else:
break
else:
self.cm._log(
f"c{self.cm.cluster_instance_num}: failed to start cluster:\n{excp}\ncluster dead"
)
if not configuration.IS_XDIST:
pytest.exit(msg=f"Failed to start cluster, exception: {excp}",
returncode=1)
helpers.touch(self.cm.instance_dir / CLUSTER_DEAD_FILE)
return False
# Create temp dir for faucet addresses data.
# Pytest's mktemp adds number to the end of the dir name, so keep the trailing '_'
# as separator. Resulting dir name is e.g. 'addrs_data_ci3_0'.
tmp_path = Path(
self.cm.tmp_path_factory.mktemp(
f"addrs_data_ci{self.cm.cluster_instance_num}_"))
# setup faucet addresses
cluster_nodes.setup_test_addrs(cluster_obj=cluster_obj,
destination_dir=tmp_path)
# create file that indicates that the cluster is running
if not cluster_running_file.exists():
helpers.touch(cluster_running_file)
return True
def stop_cluster(cmd: str) -> None:
"""Stop cluster."""
LOGGER.info(f"Stopping cluster with `{cmd}`.")
helpers.run_command(cmd, workdir=get_cluster_env().work_dir)
