def _create_from_template(template, context, logger):
"""Helper method used to generate the contents of a file given a Jinja
template and its required context.
Returns a string representing the contents of a file, or None if the
rendering from the template failed.
"""
try:
inst_dir = PbenchAgentConfig(
os.environ["_PBENCH_AGENT_CONFIG"]
).pbench_install_dir
except Exception as exc:
logger.error(
"Unexpected error encountered logging pbench agent configuration: '%s'",
exc,
)
return None
template_dir = Environment(
autoescape=False,
loader=FileSystemLoader(os.path.join(inst_dir, "templates")),
trim_blocks=False,
lstrip_blocks=False,
)
try:
filled = template_dir.get_template(template).render(context)
return filled
except Exception as exc:
logger.error("File creation failed: '%s'", exc)
return None
def config_and_logger(self, valid_config):
# Setup the configuration and logger
self.config = PbenchAgentConfig(os.environ["_PBENCH_AGENT_CONFIG"])
self.logger = get_pbench_logger("pbench", self.config)
yield
# Teardown the setup
self.config, self.logger = None, None
def config_and_logger(self):
with tempfile.TemporaryDirectory() as target_dir:
# Setup the configuration and logger
self.target_dir = target_dir
self.config = PbenchAgentConfig(os.environ["_PBENCH_AGENT_CONFIG"])
self.logger = get_pbench_logger("unittest", self.config)
yield
# Teardown the setup
self.config, self.logger = None, None
if os.path.exists(f"{os.path.realpath(MRT_DIR)}.copied"):
os.remove(f"{os.path.realpath(MRT_DIR)}.copied")
if os.path.exists(f"{os.path.realpath(MRT_DIR)}/.running"):
os.remove(f"{os.path.realpath(MRT_DIR)}/.running")
def __init__(self, controller: str, tarball: str,
config: PbenchAgentConfig, logger: Logger):
"""CopyResultTb contructor - raises FileNotFoundError if the given
tar ball does not exist, and a ValueError if the given controller is
not a valid hostname.
"""
if validate_hostname(controller) != 0:
raise ValueError(
f"Controller {controller!r} is not a valid host name")
self.controller = controller
self.tarball = Path(tarball)
if not self.tarball.exists():
raise FileNotFoundError(f"Tarball '{self.tarball}' does not exist")
server_rest_url = config.get("results", "server_rest_url")
tbname = urllib.parse.quote(self.tarball.name)
self.upload_url = f"{server_rest_url}/upload/{tbname}"
self.logger = logger
def move_results(ctx, _user, _prefix, _show_server):
config = PbenchAgentConfig(ctx["args"]["config"])
logger = get_pbench_logger("pbench", config)
controller = os.environ.get("_pbench_full_hostname")
if not controller:
logger.error("Missing controller name (should be 'hostname -f' value)")
sys.exit(1)
results_webserver = config.results.get("webserver")
if not results_webserver:
logger.error(
"No web server host configured from which we can fetch the FQDN of the host to which we copy/move results"
)
logger.debug("'webserver' variable in 'results' section not set")
server_rest_url = config.results.get("server_rest_url")
response = requests.get(f"{server_rest_url}/host_info")
if response.status_code not in [200, 201]:
logger.error(
"Unable to determine results host info from %s/host_info", server_rest_url
)
sys.exit(1)
if response.text.startswith("MESSAGE"):
message = response.text.split("===")[1]
logger.info("*** Message from sysadmins of %s:", results_webserver)
logger.info("***\n*** %s", message)
logger.info("***\n*** No local actions taken.")
sys.exit(1)
results_path_prefix = response.text.split(":")[1]
if not results_path_prefix:
logger.error(
"fetch results host info did not contain a path prefix: %s", response.text
)
sys.exit(1)
try:
temp_dir = tempfile.mkdtemp(
dir=config.pbench_tmp, prefix="pbench-move-results."
)
except Exception:
logger.error("Failed to create temporary directory")
sys.exit(1)
runs_copied = 0
failures = 0
for dirent in config.pbench_run.iterdir():
if not dirent.is_dir():
continue
if dirent.name.startswith("tools-") or dirent.name == "tmp":
continue
result_dir = dirent
mrt = MakeResultTb(result_dir, temp_dir, _user, _prefix, config, logger)
result_tb_name = mrt.make_result_tb()
assert (
result_tb_name
), "Logic bomb! make_result_tb() always returns a tar ball name"
crt = CopyResultTb(controller, result_tb_name, config, logger)
crt.copy_result_tb()
try:
# We always remove the constructed tar ball, regardless of success
# or failure, since we keep the result directory below on failure.
os.remove(result_tb_name)
os.remove(f"{result_tb_name}.md5")
except OSError:
logger.error("rm failed to remove %s and its .md5 file", result_tb_name)
sys.exit(1)
try:
shutil.rmtree(result_dir)
except OSError:
logger.error("rm failed to remove the %s directory hierarchy", result_dir)
sys.exit(1)
runs_copied += 1
if runs_copied + failures > 0:
logger.debug(
"successfully moved %s runs, encountered %s failures", runs_copied, failures
)
return runs_copied, failures
def __init__(self, context):
self.context = context
self.config = PbenchAgentConfig(self.context.config)
self.name = os.path.basename(sys.argv[0])
self.pbench_run = self.config.pbench_run
if not self.pbench_run.exists():
click.secho(
f"[ERROR] the provided pbench run directory, {self.pbench_run}, does not exist"
)
sys.exit(1)
# the pbench temporary directory is always relative to the $pbench_run
# directory
self.pbench_tmp = self.pbench_run / "tmp"
if not self.pbench_tmp.exists():
try:
os.makedirs(self.pbench_tmp)
except OSError:
click.secho(
f"[ERROR] unable to create TMP dir, {self.pbench_tmp}")
sys.exit(1)
# log file - N.B. not a directory
self.pbench_log = self.config.pbench_log
if self.pbench_log is None:
self.pbench_log = self.pbench_run / "pbench.log"
self.pbench_install_dir = self.config.pbench_install_dir
if self.pbench_install_dir is None:
self.pbench_install_dir = "/opt/pbench-agent"
if not self.pbench_install_dir.exists():
click.secho(
f"[ERROR] pbench installation directory, {self.pbench_install_dir}, does not exist"
)
sys.exit(1)
self.pbench_bspp_dir = self.pbench_install_dir / "bench-scripts" / "postprocess"
self.pbench_lib_dir = self.pbench_install_dir / "lib"
self.ssh_opts = self.config.ssh_opts
os.environ["ssh_opts"] = self.ssh_opts
self.scp_opts = self.config.scp_opts
os.environ["scp_opts"] = self.scp_opts
os.environ["_pbench_debug_mode"] = "0"
if os.environ.get("_PBENCH_UNIT_TESTS"):
self.date = "1900-01-01T00:00:00"
self.date_suffix = "1900.01.01T00.00.00"
self.hostname = "testhost"
self.full_hostname = "testhost.example.com"
else:
self.date = datetime.datetime.utcnow().strftime(
"%Y-%m-%dT%H:%M:%s")
self.date_suffix = datetime.datetime.utcnow().strftime(
"%Y-%m-%dT%H.%M.%s")
self.hostname = socket.gethostname()
self.full_hostname = socket.getfqdn()
# Backwards compatibility and for toolmeister
pbench_env = {
"date": self.date,
"date_suffix": self.date_suffix,
"hostname": self.hostname,
"full_hostname": self.full_hostname,
"pbench_run": str(self.pbench_run),
"pbench_install_dir": str(self.pbench_install_dir),
"pbench_tmp": str(self.pbench_tmp),
"pbench_log": str(self.pbench_log),
"pbench_bspp_dir": str(self.pbench_bspp_dir),
"pbench_lib_dir": str(self.pbench_lib_dir),
}
for k, v in pbench_env.items():
os.environ[k] = v
def test_invalid_config(invalid_config):
with pytest.raises(BadConfig):
PbenchAgentConfig(os.environ["_PBENCH_AGENT_CONFIG"])
def test_results_attr(valid_config):
assert ("pbench" in PbenchAgentConfig(
os.environ["_PBENCH_AGENT_CONFIG"]).results["user"])
def test_agent_attr(valid_config, pytestconfig):
TMP = pytestconfig.cache.get("TMP", None)
assert (f"{TMP}/var/lib/pbench-agent" == PbenchAgentConfig(
os.environ["_PBENCH_AGENT_CONFIG"]).agent["pbench_run"])
def test_valid_config(valid_config):
config = PbenchAgentConfig(os.environ["_PBENCH_AGENT_CONFIG"])
assert "pbench-agent" in config.conf
assert "results" in config.conf
def launch(self):
pcp_remote_config = self.benchmark_run_dir / "tm" / "remote"
with open(pcp_remote_config, "w") as remote:
for host in self.host_tools_dict:
remote.write(
f"{host} n n PCP_LOG_DIR/pmlogger/{host} -r -T24h10m -c config.{host}\n"
)
if not self.host_tools_dict:
return 0
with open("pcp.log", "w") as pcp_logs:
try:
pcp_reg = PbenchAgentConfig(
os.environ["_PBENCH_AGENT_CONFIG"]
).pmlogger_reg
except Exception as exc:
self.logger.error(
"Unexpected error encountered logging pbench agent configuration: '%s'",
exc,
)
return 0
args = ["podman", "pull", pcp_reg]
try:
pcp_pull = subprocess.Popen(args, stdout=pcp_logs, stderr=pcp_logs)
pcp_pull.wait()
except Exception as exc:
self.logger.error("Podman pull process failed: '%s'", exc)
return 0
try:
os.mkdir(self.volume)
os.chmod(self.volume, 0o777)
except Exception as exc:
self.logger.error("Volume creation failed: '%s'", exc)
return 0
self.podname = "collector" + datetime.now().strftime("%m-%d-%y-%H-%M-%S")
args = [
"podman",
"run",
"--systemd",
"always",
"-v",
f"{self.volume}:/var/log/pcp/pmlogger:Z",
"-v",
f"{pcp_remote_config}:/etc/pcp/pmlogger/control.d/remote:Z",
"--network",
"host",
"--name",
self.podname,
pcp_reg,
]
try:
self.__test_conn()
self.run = subprocess.Popen(args, stdout=pcp_logs, stderr=pcp_logs)
except Exception as exc:
self.logger.error("Podman run process failed: '%s'", exc)
self.run = None
self.podname = None
return 0
return 1
def launch(self):
tool_context = []
with open("prometheus.yml", "w") as config:
for host in self.host_tools_dict:
for tool in self.host_tools_dict[host]:
tool_dict = {}
port = self.tool_metadata.getProperties(tool)["port"]
tool_dict["hostname"] = host + "_" + tool
tool_dict["hostport"] = host + ":" + port
tool_context.append(tool_dict)
if tool_context:
tool_context = {"tools": tool_context}
yml = _create_from_template("prometheus.yml", tool_context, self.logger)
config.write(yml)
else:
return 0
with open("prom.log", "w") as prom_logs:
try:
prom_reg = PbenchAgentConfig(
os.environ["_PBENCH_AGENT_CONFIG"]
).prom_reg
except Exception as exc:
self.logger.error(
"Unexpected error encountered logging pbench agent configuration: '%s'",
exc,
)
return 0
args = ["podman", "pull", prom_reg]
try:
prom_pull = subprocess.Popen(args, stdout=prom_logs, stderr=prom_logs)
prom_pull.wait()
except Exception as exc:
self.logger.error("Podman pull process failed: '%s'", exc)
return 0
try:
os.mkdir(self.volume)
os.chmod(self.volume, 0o777)
except Exception as exc:
self.logger.error("Volume creation failed: '%s'", exc)
return 0
args = [
"podman",
"run",
"-p",
"9090:9090",
"-v",
f"{self.volume}:/prometheus:Z",
"-v",
f"{self.benchmark_run_dir}/tm/prometheus.yml:/etc/prometheus/prometheus.yml:Z",
"--network",
"host",
prom_reg,
]
try:
self.run = subprocess.Popen(args, stdout=prom_logs, stderr=prom_logs)
except Exception as exc:
self.logger.error("Podman run process failed: '%s'", exc)
self.run = None
return 0
return 1
def move_results(ctx, _user, _prefix, _show_server):
config = PbenchAgentConfig(ctx["args"]["config"])
logger = get_pbench_logger("pbench-move-results", config)
controller = os.environ.get("full_hostname")
if not controller:
logger.error("Missing controller name (should be 'hostname -f' value)")
sys.exit(1)
results_webserver = config.results.get("webserver")
if not results_webserver:
logger.error(
"No web server host configured from which we can fetch the FQDN of the host to which we copy/move results"
)
logger.debug("'webserver' variable in 'results' section not set")
if not _user:
_user = config.agent.get("pbench_user")
server_rest_url = config.results.get("server_rest_url")
response = requests.get(f"{server_rest_url}/host_info")
if response.status_code not in [200, 201]:
logger.error("Unable to determine results host info from %s/host_info",
server_rest_url)
sys.exit(1)
if response.text.startswith("MESSAGE"):
message = response.text.split("===")[1]
logger.info("*** Message from sysadmins of %s:", results_webserver)
logger.info("***\n*** %s", message)
logger.info("***\n*** No local actions taken.")
sys.exit(1)
results_path_prefix = response.text.split(":")[1]
if not results_path_prefix:
logger.error(
"fetch results host info did not contain a path prefix: %s",
response.text)
sys.exit(1)
runs_copied = 0
failures = 0
try:
temp_dir = tempfile.mkdtemp(dir=config.pbench_tmp,
prefix="pbench-move-results.")
except Exception:
logger.error("Failed to create temporary directory")
sys.exit(1)
dirs = [
_dir for _dir in next(os.walk(config.pbench_run))[1]
if not _dir.startswith("tools-") and not _dir.startswith("tmp")
]
for _dir in dirs:
result_dir = config.pbench_run / _dir
mrt = MakeResultTb(result_dir, temp_dir, _user, _prefix, config,
logger)
result_tb_name = mrt.make_result_tb()
if result_tb_name:
crt = CopyResultTb(controller, result_tb_name, config, logger)
copy_result = crt.copy_result_tb()
try:
os.remove(result_tb_name)
os.remove(f"{result_tb_name}.md5")
except OSError:
logger.error("rm failed to remove %s and its .md5 file",
result_tb_name)
sys.exit(1)
if not copy_result:
failures += 1
continue
try:
os.remove(result_dir)
except OSError:
logger.error("rm failed to remove the %s directory hierarchy",
result_dir)
sys.exit(1)
runs_copied += 1
if runs_copied + failures > 0:
logger.debug("successfully moved %s runs, encountered %s failures",
runs_copied, failures)
return failures
