def test_make_tb(self, monkeypatch):
monkeypatch.setattr(datetime, "datetime", MockDatetime)
mrt = MakeResultTb(MRT_DIR, self.target_dir, self.config, self.logger)
tarball = mrt.make_result_tb()
expected_tb = os.path.join(self.target_dir, "make_result_tb.tar.xz")
assert Path(tarball).samefile(expected_tb)
assert os.path.exists(tarball)
def test_already_copied(self, caplog):
caplog.set_level(logging.DEBUG, logger=self.logger.name)
full_result_dir = os.path.realpath(MRT_DIR)
expected_error_message = f"Already copied {full_result_dir}"
with open(f"{full_result_dir}.copied", "x"):
with pytest.raises(Exception) as e:
mrt = MakeResultTb(MRT_DIR, self.target_dir, self.config, self.logger)
mrt.make_result_tb()
assert str(e).endswith(expected_error_message)
def test_make_tb(self, monkeypatch):
monkeypatch.setattr(datetime, "datetime", MockDatetime)
try:
mrt = MakeResultTb(MRT_DIR, self.target_dir, "pbench", "",
self.config, self.logger)
tarball = mrt.make_result_tb()
except SystemExit:
assert False
assert tarball == os.path.join(self.target_dir,
"make_result_tb.tar.xz")
assert os.path.exists(tarball)
def test_running(self, caplog):
caplog.set_level(logging.DEBUG, logger=self.logger.name)
expected_error_message = (
f"The benchmark is still running in {os.path.basename(MRT_DIR)} - skipping;"
f" if that is not true, rmdir {os.path.basename(MRT_DIR)}/.running, and try again"
)
full_result_dir = os.path.realpath(MRT_DIR)
with open(f"{full_result_dir}/.running", "x"):
with pytest.raises(RuntimeError) as e:
mrt = MakeResultTb(MRT_DIR, self.target_dir, self.config, self.logger)
mrt.make_result_tb()
assert str(e).endswith(expected_error_message)
def test_running(self, caplog):
caplog.set_level(logging.DEBUG, logger=self.logger.name)
full_result_dir = os.path.realpath(MRT_DIR)
expected_error_message = f"Already copied {full_result_dir}"
with open(f"{full_result_dir}.copied", "x"):
try:
mrt = MakeResultTb(MRT_DIR, self.target_dir, "pbench", "",
self.config, self.logger)
mrt.make_result_tb()
except SystemExit:
assert caplog.records
assert caplog.records[0].message == expected_error_message
else:
assert False
def test_bad_target_dir(self, target_dir, caplog):
caplog.set_level(logging.ERROR, logger=self.logger.name)
if target_dir:
expected_error_message = f"Invalid target directory provided: {target_dir}"
else:
expected_error_message = "Target directory not provided"
try:
mrt = MakeResultTb(MRT_DIR, target_dir, "pbench", "", self.config,
self.logger)
mrt.make_result_tb()
except SystemExit:
assert caplog.records
assert len(caplog.records) == 1
assert caplog.records[0].message == expected_error_message
else:
assert False
def test_already_copied(self, caplog):
caplog.set_level(logging.DEBUG, logger=self.logger.name)
expected_error_message = (
f"The benchmark is still running in {os.path.basename(MRT_DIR)} - skipping;"
f" if that is not true, rmdir {os.path.basename(MRT_DIR)}/.running, and try again"
)
full_result_dir = os.path.realpath(MRT_DIR)
with open(f"{full_result_dir}/.running", "x"):
try:
mrt = MakeResultTb(MRT_DIR, self.target_dir, "pbench", "",
self.config, self.logger)
mrt.make_result_tb()
except SystemExit:
assert caplog.records
assert len(caplog.records) == 1
assert caplog.records[0].message == expected_error_message
else:
assert False
def move_results(ctx, _user, _prefix, _show_server):
config = PbenchAgentConfig(ctx["args"]["config"])
logger = get_pbench_logger("pbench", config)
controller = os.environ.get("_pbench_full_hostname")
if not controller:
logger.error("Missing controller name (should be 'hostname -f' value)")
sys.exit(1)
results_webserver = config.results.get("webserver")
if not results_webserver:
logger.error(
"No web server host configured from which we can fetch the FQDN of the host to which we copy/move results"
)
logger.debug("'webserver' variable in 'results' section not set")
server_rest_url = config.results.get("server_rest_url")
response = requests.get(f"{server_rest_url}/host_info")
if response.status_code not in [200, 201]:
logger.error(
"Unable to determine results host info from %s/host_info", server_rest_url
)
sys.exit(1)
if response.text.startswith("MESSAGE"):
message = response.text.split("===")[1]
logger.info("*** Message from sysadmins of %s:", results_webserver)
logger.info("***\n*** %s", message)
logger.info("***\n*** No local actions taken.")
sys.exit(1)
results_path_prefix = response.text.split(":")[1]
if not results_path_prefix:
logger.error(
"fetch results host info did not contain a path prefix: %s", response.text
)
sys.exit(1)
try:
temp_dir = tempfile.mkdtemp(
dir=config.pbench_tmp, prefix="pbench-move-results."
)
except Exception:
logger.error("Failed to create temporary directory")
sys.exit(1)
runs_copied = 0
failures = 0
for dirent in config.pbench_run.iterdir():
if not dirent.is_dir():
continue
if dirent.name.startswith("tools-") or dirent.name == "tmp":
continue
result_dir = dirent
mrt = MakeResultTb(result_dir, temp_dir, _user, _prefix, config, logger)
result_tb_name = mrt.make_result_tb()
assert (
result_tb_name
), "Logic bomb! make_result_tb() always returns a tar ball name"
crt = CopyResultTb(controller, result_tb_name, config, logger)
crt.copy_result_tb()
try:
# We always remove the constructed tar ball, regardless of success
# or failure, since we keep the result directory below on failure.
os.remove(result_tb_name)
os.remove(f"{result_tb_name}.md5")
except OSError:
logger.error("rm failed to remove %s and its .md5 file", result_tb_name)
sys.exit(1)
try:
shutil.rmtree(result_dir)
except OSError:
logger.error("rm failed to remove the %s directory hierarchy", result_dir)
sys.exit(1)
runs_copied += 1
if runs_copied + failures > 0:
logger.debug(
"successfully moved %s runs, encountered %s failures", runs_copied, failures
)
return runs_copied, failures
def test_bad_target_dir(self, target_dir, caplog):
caplog.set_level(logging.ERROR, logger=self.logger.name)
with pytest.raises(FileNotFoundError):
mrt = MakeResultTb(MRT_DIR, target_dir, self.config, self.logger)
mrt.make_result_tb()
def move_results(ctx, _user, _prefix, _show_server):
config = PbenchAgentConfig(ctx["args"]["config"])
logger = get_pbench_logger("pbench-move-results", config)
controller = os.environ.get("full_hostname")
if not controller:
logger.error("Missing controller name (should be 'hostname -f' value)")
sys.exit(1)
results_webserver = config.results.get("webserver")
if not results_webserver:
logger.error(
"No web server host configured from which we can fetch the FQDN of the host to which we copy/move results"
)
logger.debug("'webserver' variable in 'results' section not set")
if not _user:
_user = config.agent.get("pbench_user")
server_rest_url = config.results.get("server_rest_url")
response = requests.get(f"{server_rest_url}/host_info")
if response.status_code not in [200, 201]:
logger.error("Unable to determine results host info from %s/host_info",
server_rest_url)
sys.exit(1)
if response.text.startswith("MESSAGE"):
message = response.text.split("===")[1]
logger.info("*** Message from sysadmins of %s:", results_webserver)
logger.info("***\n*** %s", message)
logger.info("***\n*** No local actions taken.")
sys.exit(1)
results_path_prefix = response.text.split(":")[1]
if not results_path_prefix:
logger.error(
"fetch results host info did not contain a path prefix: %s",
response.text)
sys.exit(1)
runs_copied = 0
failures = 0
try:
temp_dir = tempfile.mkdtemp(dir=config.pbench_tmp,
prefix="pbench-move-results.")
except Exception:
logger.error("Failed to create temporary directory")
sys.exit(1)
dirs = [
_dir for _dir in next(os.walk(config.pbench_run))[1]
if not _dir.startswith("tools-") and not _dir.startswith("tmp")
]
for _dir in dirs:
result_dir = config.pbench_run / _dir
mrt = MakeResultTb(result_dir, temp_dir, _user, _prefix, config,
logger)
result_tb_name = mrt.make_result_tb()
if result_tb_name:
crt = CopyResultTb(controller, result_tb_name, config, logger)
copy_result = crt.copy_result_tb()
try:
os.remove(result_tb_name)
os.remove(f"{result_tb_name}.md5")
except OSError:
logger.error("rm failed to remove %s and its .md5 file",
result_tb_name)
sys.exit(1)
if not copy_result:
failures += 1
continue
try:
os.remove(result_dir)
except OSError:
logger.error("rm failed to remove the %s directory hierarchy",
result_dir)
sys.exit(1)
runs_copied += 1
if runs_copied + failures > 0:
logger.debug("successfully moved %s runs, encountered %s failures",
runs_copied, failures)
return failures
def execute(self) -> int:
logger = get_pbench_logger("pbench-agent", self.config)
temp_dir = tempfile.mkdtemp(dir=self.config.pbench_tmp,
prefix="pbench-move-results.")
runs_copied = 0
failures = 0
no_of_tb = 0
for dirent in self.config.pbench_run.iterdir():
if not dirent.is_dir():
continue
if dirent.name.startswith("tools-") or dirent.name == "tmp":
continue
no_of_tb += 1
result_dir = dirent
try:
mrt = MakeResultTb(result_dir, temp_dir, self.config, logger)
except FileNotFoundError as e:
logger.error("File Not Found Error, {}", e)
continue
except NotADirectoryError as e:
logger.error("Bad Directory, {}", e)
continue
try:
result_tb_name = mrt.make_result_tb()
except BadMDLogFormat as e:
logger.warning("Bad Metadata.log file encountered, {}", e)
failures += 1
continue
except FileNotFoundError as e:
logger.debug("File Not Found error, {}", e)
failures += 1
continue
except RuntimeError as e:
logger.warning("Unexpected Error encountered, {}", e)
failures += 1
continue
except Exception as e:
logger.debug("Unexpected Error occurred, {}", e)
failures += 1
continue
try:
crt = CopyResultTb(self.context.controller, result_tb_name,
self.config, logger)
except FileNotFoundError as e:
logger.error("File Not Found error, {}", e)
failures += 1
continue
try:
crt.copy_result_tb(self.context.token)
except (FileUploadError, RuntimeError) as e:
logger.error("Error uploading a file, {}", e)
failures += 1
continue
try:
# We always remove the constructed tar ball, regardless of success
# or failure, since we keep the result directory below on failure.
os.remove(result_tb_name)
except OSError:
logger.error("Failed to remove {}", result_tb_name)
failures += 1
continue
try:
shutil.rmtree(result_dir)
except OSError:
logger.error("Failed to remove the {} directory hierarchy",
result_dir)
failures += 1
continue
runs_copied += 1
logger.info(
"Status: Total no. of tarballs {}, Successfully moved {}, Encountered {} failures",
no_of_tb,
runs_copied,
failures,
)
return 0