def _get_conf(self, section, option):
"""
_get_conf - get the option from the section, raising
BadConfig Error if it is empty, return the option value as a string.
"""
try:
option_val = self.conf.get(section, option)
except (NoOptionError, NoSectionError) as exc:
raise BadConfig(str(exc))
else:
if not option_val:
raise BadConfig(f"option {option} in section {section} is empty")
return option_val
def __init__(self, cfg_name):
# Enumerate the list of files
config_files = configtools.file_list(cfg_name)
config_files.reverse()
self.conf = ConfigParser()
self.files = self.conf.read(config_files)
try:
self.logger_type = self.conf.get("logging", "logger_type")
except (NoOptionError, NoSectionError):
self.logger_type = "devlog"
else:
if self.logger_type == "hostport":
try:
self.logger_host = self.conf.get("logging", "logger_host")
self.logger_port = self.conf.get("logging", "logger_port")
except (NoOptionError) as exc:
raise BadConfig(str(exc))
try:
self.default_logging_level = self.conf.get("logging",
"logging_level")
except (NoOptionError, NoSectionError):
self.default_logging_level = "INFO"
# Constants
# Force UTC everywhere
self.TZ = "UTC"
def _get_valid_dir_option(self, req_val, section, option):
"""_get_valid_dir_option - get the directory option from the
given section, raising BadConfig Error if the path is not resolved
or is not a directory, returning a Path directory object.
"""
dir_val = self._get_conf(section, option)
dir_path = self._get_valid_path(req_val, dir_val, None)
if not dir_path:
raise BadConfig(f"Bad {req_val}={dir_val}")
return dir_path
def __init__(self, cfg_name):
# Enumerate the list of files
config_files = configtools.file_list(cfg_name)
config_files.reverse()
self.conf = ConfigParser()
self.files = self.conf.read(config_files)
try:
self.logger_type = self.conf.get("logging", "logger_type")
except (NoOptionError, NoSectionError):
self.logger_type = "devlog"
else:
if self.logger_type == "hostport":
try:
self.logger_host = self.conf.get("logging", "logger_host")
self.logger_port = self.conf.get("logging", "logger_port")
except (NoOptionError) as exc:
raise BadConfig(str(exc))
try:
self.log_dir = self.conf.get("logging", "log_dir")
except (NoOptionError, NoSectionError):
# Allow for compatibility with previous configuration files. Any
# sub-class of this base class can change the log directory as
# necessary. Note that this class ONLY records the value found in
# the configuration file, it does not take any actions on it.
self.log_dir = None
# The Agent and Server classes behave a bit differently with respect
# to logging when using the "file" logger type. While this is not
# derived from a configuration file, we keep track of the behavioral
# attribute with the configuration. The sub-classes will override the
# value as necessary.
self.log_using_caller_directory = False
try:
self.default_logging_level = self.conf.get("logging",
"logging_level")
except (NoOptionError, NoSectionError):
self.default_logging_level = "INFO"
try:
# We don't document the "log_format" parameter since it is really
# only present to facilitate easier unit testing.
self.log_fmt = self.conf.get("logging", "log_format")
except (NoOptionError, NoSectionError):
self.log_fmt = None
# Constants
# Force UTC everywhere
self.TZ = "UTC"
def get_pbench_logger(caller, config):
"""Fetch the logger specifed by "caller", and add a specific handler
based on the logging configuration requested.
We also return a logger that supports "brace" style message formatting,
e.g. logger.warning("that = {}", that)
"""
pbench_logger = logging.getLogger(caller)
if caller not in _handlers:
try:
logging_level = config.get(caller, "logging_level")
except (NoSectionError, NoOptionError):
logging_level = config.default_logging_level
pbench_logger.setLevel(logging_level)
if config.logger_type == "file":
log_dir = Path(config.log_dir)
if config.log_using_caller_directory:
log_dir = log_dir / caller
try:
log_dir.mkdir()
except FileExistsError:
# directory already exists, ignore
pass
handler = logging.FileHandler(log_dir / f"{caller}.log")
elif config.logger_type == "devlog":
handler = logging.handlers.SysLogHandler(address=_devlog)
elif (config.logger_type == "hostport"
): # hostport logger type uses UDP-based logging
handler = logging.handlers.SysLogHandler(
address=(config.logger_host, int(config.logger_port)))
else:
raise BadConfig("Unsupported logger type")
handler.setLevel(logging.DEBUG)
if config.log_fmt is None:
logfmt = "{asctime} {levelname} {process} {thread} {name}.{module} {funcName} {lineno} -- {message}"
else:
logfmt = config.log_fmt
formatter = _PbenchLogFormatter(fmt=logfmt)
handler.setFormatter(formatter)
_handlers[caller] = handler
pbench_logger.addHandler(handler)
return _StyleAdapter(pbench_logger)
def __init__(self, cfg_name):
super().__init__(cfg_name)
try:
# Provide a few convenience attributes.
self.agent = self.conf["pbench-agent"]
self.results = self.conf["results"]
# Now fetch some default common pbench settings that are required.
self.pbench_run = Path(
self.conf.get("pbench-agent",
"pbench_run",
fallback=DEFAULT_PBENCH_AGENT_RUN_DIR))
self.pbench_tmp = self.pbench_run / "tmp"
self.pbench_log = Path(
self.conf.get(
"pbench-agent",
"pbench_log",
fallback=str(self.pbench_run / "pbench.log"),
))
self.pbench_install_dir = Path(
self.conf.get(
"pbench-agent",
"install-dir",
fallback=DEFAULT_PBENCH_AGENT_INSTALL_DIR,
))
except (NoOptionError, NoSectionError, KeyError) as exc:
raise BadConfig(f"{exc}: {self.files}")
else:
if not self.pbench_install_dir.is_dir():
raise BadConfig(
"pbench installation directory,"
f" '{self.pbench_install_dir}', does not exist")
try:
self.pbench_tmp.mkdir(parents=True, exist_ok=True)
except OSError as exc:
raise BadConfig(str(exc))
self.pbench_bspp_dir = (self.pbench_install_dir / "bench-scripts" /
"postprocess")
self.pbench_lib_dir = self.pbench_install_dir / "lib"
if self.logger_type == "file" and self.log_dir is None:
# The configuration file has a logging section configured to use
# "file" logging, but no log directory is set. We'll set the log
# directory to be the directory of the legacy ${pbench_log} value
# determined above.
self.log_dir = str(self.pbench_log.parent)
try:
self.ssh_opts = self.conf.get("results",
"ssh_opts",
fallback=DEFAULT_SSH_OPTS)
except (NoOptionError, NoSectionError):
self.ssh_opts = DEFAULT_SSH_OPTS
try:
self.scp_opts = self.conf.get("results",
"scp_opts",
fallback=DEFAULT_SCP_OPTS)
except (NoOptionError, NoSectionError):
self.scp_opts = DEFAULT_SCP_OPTS
try:
self.prom_reg = self.conf.get("container-registry",
"prometheus",
fallback=DEFAULT_PROM_REG)
except (NoOptionError, NoSectionError):
self.prom_reg = DEFAULT_PROM_REG
try:
self.pmlogger_reg = self.conf.get("container-registry",
"pcp_pmlogger",
fallback=DEFAULT_PMLOGGER_REG)
except (NoOptionError, NoSectionError):
self.pmlogger_reg = DEFAULT_PMLOGGER_REG
try:
self.pmcd_reg = self.conf.get("container-registry",
"pcp_pmcd",
fallback=DEFAULT_PMCD_REG)
except (NoOptionError, NoSectionError):
self.pmcd_reg = DEFAULT_PMCD_REG
try:
self._unittests = self.conf.get("pbench-agent", "debug_unittest")
except Exception:
self._unittests = False
else:
self._unittests = bool(self._unittests)
try:
self._debug = self.conf.get("pbench-agent", "debug")
except Exception:
self._debug = False
else:
self._debug = bool(self._debug)
def __init__(self, cfg_name):
super().__init__(cfg_name)
try:
# Provide a few convenience attributes.
self.agent = self.conf["pbench-agent"]
self.results = self.conf["results"]
# Now fetch some default common pbench settings that are required.
self.pbench_run = Path(
self.conf.get("pbench-agent",
"pbench_run",
fallback=DEFAULT_PBENCH_AGENT_RUN_DIR))
self.pbench_tmp = self.pbench_run / "tmp"
self.pbench_log = Path(
self.conf.get(
"pbench-agent",
"pbench_log",
fallback=str(self.pbench_run / "pbench.log"),
))
self.pbench_install_dir = Path(
self.conf.get(
"pbench-agent",
"install-dir",
fallback=DEFAULT_PBENCH_AGENT_INSTALL_DIR,
))
except (NoOptionError, NoSectionError, KeyError) as exc:
raise BadConfig(f"{exc}: {self.files}")
else:
if not self.pbench_install_dir.is_dir():
raise BadConfig(
"pbench installation directory,"
f" '{self.pbench_install_dir}', does not exist")
try:
self.pbench_tmp.mkdir(parents=True, exist_ok=True)
except OSError as exc:
raise BadConfig(str(exc))
self.pbench_bspp_dir = (self.pbench_install_dir / "bench-scripts" /
"postprocess")
self.pbench_lib_dir = self.pbench_install_dir / "lib"
self.LOGSDIR = self.pbench_log
try:
self.ssh_opts = self.conf.get("results",
"ssh_opts",
fallback=DEFAULT_SSH_OPTS)
except (NoOptionError, NoSectionError):
self.ssh_opts = DEFAULT_SSH_OPTS
try:
self.scp_opts = self.conf.get("results",
"scp_opts",
fallback=DEFAULT_SCP_OPTS)
except (NoOptionError, NoSectionError):
self.scp_opts = DEFAULT_SCP_OPTS
try:
self._unittests = self.conf.get("pbench-agent", "debug_unittest")
except Exception:
self._unittests = False
else:
self._unittests = bool(self._unittests)
try:
self._debug = self.conf.get("pbench-agent", "debug")
except Exception:
self._debug = False
else:
self._debug = bool(self._debug)
def __init__(self, cfg_name):
super().__init__(cfg_name)
# Now fetch some default common pbench settings that are required.
try:
self.TOP = Path(self.conf.get("pbench-server", "pbench-top-dir"))
if not self.TOP.is_dir():
raise BadConfig(f"Bad TOP={self.TOP}")
self.TMP = Path(self.conf.get("pbench-server", "pbench-tmp-dir"))
if not self.TMP.is_dir():
raise BadConfig(f"Bad TMP={self.TMP}")
self.LOGSDIR = Path(
self.conf.get("pbench-server", "pbench-logs-dir"))
if not self.LOGSDIR.is_dir():
raise BadConfig(f"Bad LOGSDIR={self.LOGSDIR}")
self.BINDIR = Path(self.conf.get("pbench-server", "script-dir"))
if not self.BINDIR.is_dir():
raise BadConfig(f"Bad BINDIR={self.BINDIR}")
self.LIBDIR = Path(self.conf.get("pbench-server", "lib-dir"))
if not self.LIBDIR.is_dir():
raise BadConfig(f"Bad LIBDIR={self.LIBDIR}")
# the scripts may use this to send status messages
self.mail_recipients = self.conf.get("pbench-server", "mailto")
self.ARCHIVE = Path(
self.conf.get("pbench-server", "pbench-archive-dir"))
except (NoOptionError, NoSectionError) as exc:
raise BadConfig(str(exc))
else:
self.INCOMING = self.TOP / "public_html" / "incoming"
# this is where the symlink forest is going to go
self.RESULTS = self.TOP / "public_html" / "results"
self.USERS = self.TOP / "public_html" / "users"
try:
self.PBENCH_ENV = self.conf.get("pbench-server", "environment")
except NoOptionError:
self.PBENCH_ENV = ""
try:
self.COMMIT_ID = self.conf.get("pbench-server", "commit_id")
except NoOptionError:
self.COMMIT_ID = "(unknown)"
try:
self._unittests = self.conf.get("pbench-server", "debug_unittest")
except Exception:
self._unittests = False
else:
self._unittests = bool(self._unittests)
if self._unittests:
def mocked_time():
return 42.00
global _time
_time = mocked_time
try:
ref_dt_str = self.conf.get("pbench-server",
"debug_ref_datetime")
except Exception:
ref_dt_str = "1970-01-02T00:00:00.000000"
self._ref_datetime = datetime.strptime(ref_dt_str,
_STD_DATETIME_FMT)
else:
self._ref_datetime = None
# Constants
# Initial common timestamp format
self.TS = f"run-{self.timestamp()}"
# Make all the state directories for the pipeline and any others
# needed. Every related state directories are paired together with
# their final state at the end.
self.LINKDIRS = (
"TODO BAD-MD5"
" TO-UNPACK UNPACKED WONT-UNPACK"
" TO-SYNC SYNCED"
" TO-LINK"
" TO-INDEX TO-RE-INDEX TO-INDEX-TOOL INDEXED WONT-INDEX"
" TO-COPY-SOS COPIED-SOS"
" TO-BACKUP BACKED-UP BACKUP-FAILED"
" SATELLITE-MD5-PASSED SATELLITE-MD5-FAILED"
" TO-DELETE SATELLITE-DONE")
# List of the state directories which will be excluded during rsync.
# Note that range(1,12) generates the sequence [1..11] inclusively.
self.EXCLUDE_DIRS = (
"_QUARANTINED " + self.LINKDIRS + " " +
" ".join([f"WONT-INDEX.{i:d}" for i in range(1, 12)]))