def read_json_config_file(filename,
*,
check_commands=True,
add_defaults=True,
check_pgdata=True):
try:
with open(filename, "r") as fp:
config = json.load(fp)
except FileNotFoundError:
raise InvalidConfigurationError(
"Configuration file {!r} does not exist".format(filename))
except ValueError as ex:
raise InvalidConfigurationError(
"Configuration file {!r} does not contain valid JSON: {}".format(
filename, str(ex)))
except OSError as ex:
raise InvalidConfigurationError(
"Configuration file {!r} can't be opened: {}".format(
filename, ex.__class__.__name__))
if not add_defaults:
return config
return set_and_check_config_defaults(config,
check_commands=check_commands,
check_pgdata=check_pgdata)
def get_site_from_config(config, site):
if not config.get("backup_sites"):
raise InvalidConfigurationError("No backup sites defined in configuration")
site_count = len(config["backup_sites"])
if site is None:
if site_count > 1:
raise InvalidConfigurationError("Backup site not set and configuration file defines {} sites: {}"
.format(site_count, sorted(config["backup_sites"])))
site = list(config["backup_sites"])[0]
elif site not in config["backup_sites"]:
n_sites = "{} other site{}".format(site_count, "s" if site_count > 1 else "")
raise InvalidConfigurationError("Site {!r} not defined in configuration file. {} are defined: {}"
.format(site, n_sites, sorted(config["backup_sites"])))
return site
def load_config(self, _signal=None, _frame=None): # pylint: disable=unused-argument
self.log.debug("Loading JSON config from: %r, signal: %r", self.config_path, _signal)
try:
new_config = config.read_json_config_file(self.config_path)
except (InvalidConfigurationError, subprocess.CalledProcessError, UnicodeDecodeError) as ex:
self.log.exception("Invalid config file %r: %s: %s", self.config_path, ex.__class__.__name__, ex)
# if we were called by a signal handler we'll ignore (and log)
# the error and hope the user fixes the configuration before
# restarting pghoard.
if _signal is not None:
return
if isinstance(ex, InvalidConfigurationError):
raise
raise InvalidConfigurationError(self.config_path)
self.config = new_config
if self.config.get("syslog") and not self.syslog_handler:
self.syslog_handler = logutil.set_syslog_handler(
address=self.config.get("syslog_address", "/dev/log"),
facility=self.config.get("syslog_facility", "local2"),
logger=logging.getLogger(),
)
# NOTE: getLevelName() also converts level names to numbers
self.log_level = logging.getLevelName(self.config["log_level"])
try:
logging.getLogger().setLevel(self.log_level)
except ValueError:
self.log.exception("Problem with log_level: %r", self.log_level)
# statsd settings may have changed
stats = self.config.get("statsd", {})
self.stats = statsd.StatsClient(host=stats.get("host"), port=stats.get("port"),
tags=stats.get("tags"), message_format=stats.get("format"))
self.log.debug("Loaded config: %r from: %r", self.config, self.config_path)
def compressor(self, compression_algorithm):
if compression_algorithm == "lzma":
return lzma.LZMACompressor(preset=0)
elif compression_algorithm == "snappy":
if not snappy:
raise MissingLibraryError(
"python-snappy is required when using snappy compression")
return snappy.StreamCompressor()
else:
raise InvalidConfigurationError(
"invalid compression algorithm: {!r}".format(
compression_algorithm))
def decompressor(self, algorithm):
if algorithm is None:
return None
if algorithm == "lzma":
return lzma.LZMADecompressor()
elif algorithm == "snappy":
if not snappy:
raise MissingLibraryError(
"python-snappy is required when using snappy compression")
return snappy.StreamDecompressor()
else:
raise InvalidConfigurationError(
"invalid compression algorithm: {!r}".format(algorithm))
def load_config(self, _signal=None, _frame=None): # pylint: disable=unused-argument
self.log.debug("Loading JSON config from: %r, signal: %r",
self.config_path, _signal)
try:
new_config = config.read_json_config_file(self.config_path)
except (InvalidConfigurationError, subprocess.CalledProcessError,
UnicodeDecodeError) as ex:
self.log.exception("Invalid config file %r: %s: %s",
self.config_path, ex.__class__.__name__, ex)
# if we were called by a signal handler we'll ignore (and log)
# the error and hope the user fixes the configuration before
# restarting pghoard.
if _signal is not None:
return
if isinstance(ex, InvalidConfigurationError):
raise
raise InvalidConfigurationError(self.config_path)
# clear this objects site transfer storage config
self.site_transfers = {}
self.config = new_config
if self.config.get("syslog") and not self.syslog_handler:
self.syslog_handler = logutil.set_syslog_handler(
address=self.config.get("syslog_address", "/dev/log"),
facility=self.config.get("syslog_facility", "local2"),
logger=logging.getLogger(),
)
# NOTE: getLevelName() also converts level names to numbers
self.log_level = logging.getLevelName(self.config["log_level"])
try:
logging.getLogger().setLevel(self.log_level)
except ValueError:
self.log.exception("Problem with log_level: %r", self.log_level)
# Setup monitoring clients
self.metrics = metrics.Metrics(
statsd=self.config.get("statsd", None),
pushgateway=self.config.get("pushgateway", None),
prometheus=self.config.get("prometheus", None))
# need to refresh the web server config too
if hasattr(self, "webserver") and hasattr(self.webserver, "server"):
self.webserver.server.config = new_config
for thread in self._get_all_threads():
thread.config = new_config
thread.site_transfers = {}
self.log.debug("Loaded config: %r from: %r", self.config,
self.config_path)
def get_object_storage_config(config, site):
try:
storage_config = config["backup_sites"][site]["object_storage"]
except KeyError:
# fall back to `local` driver at `backup_location` if set
if not config["backup_location"]:
return None
storage_config = {
"directory": config["backup_location"],
"storage_type": "local",
}
if "storage_type" not in storage_config:
raise InvalidConfigurationError("storage_type not defined in site {!r} object_storage".format(site))
return storage_config
def set_config_defaults(config, *, check_commands=True):
# TODO: consider implementing a real configuration schema at some point
# misc global defaults
config.setdefault("backup_location", None)
config.setdefault("http_address", PGHOARD_HOST)
config.setdefault("http_port", PGHOARD_PORT)
config.setdefault("alert_file_dir",
config.get("backup_location") or os.getcwd())
config.setdefault("json_state_file_path",
"/var/lib/pghoard/pghoard_state.json")
config.setdefault("maintenance_mode_file",
"/var/lib/pghoard/maintenance_mode_file")
config.setdefault("log_level", "INFO")
config.setdefault("path_prefix", "")
config.setdefault("upload_retries_warning_limit", 3)
# default to 5 compression and transfer threads
config.setdefault("compression", {}).setdefault("thread_count",
max(get_cpu_count(), 5))
config.setdefault("transfer", {}).setdefault("thread_count",
max(get_cpu_count(), 5))
# default to prefetching min(#compressors, #transferagents) - 1 objects so all
# operations where prefetching is used run fully in parallel without waiting to start
config.setdefault(
"restore_prefetch",
min(config["compression"]["thread_count"],
config["transfer"]["thread_count"]) - 1)
# if compression algorithm is not explicitly set prefer snappy if it's available
if snappy is not None:
config["compression"].setdefault("algorithm", "snappy")
else:
config["compression"].setdefault("algorithm", "lzma")
config["compression"].setdefault("level", 0)
# defaults for sites
config.setdefault("backup_sites", {})
for site_name, site_config in config["backup_sites"].items():
site_config.setdefault("active", True)
site_config.setdefault("active_backup_mode", "pg_receivexlog")
site_config.setdefault("basebackup_count", 2)
site_config.setdefault("basebackup_interval_hours", 24)
site_config.setdefault(
"basebackup_mode",
"pipe" if site_config.get("stream_compression") else "basic")
site_config.setdefault("encryption_key_id", None)
site_config.setdefault("object_storage", None)
# NOTE: pg_data_directory doesn't have a default value
data_dir = site_config.get("pg_data_directory")
if not data_dir and site_config["basebackup_mode"] == "local-tar":
raise InvalidConfigurationError(
"Site {!r}: pg_data_directory must be set to use `local-tar` backup mode"
.format(site_name))
version_file = os.path.join(data_dir,
"PG_VERSION") if data_dir else None
if version_file and os.path.exists(version_file):
with open(version_file, "r") as fp:
site_config["pg_data_directory_version"] = fp.read().strip()
else:
site_config["pg_data_directory_version"] = None
# FIXME: pg_xlog_directory has historically had a default value, but we should probably get rid of it
# as an incorrect value here will have unfortunate consequences. Also, since we now have a
# pg_data_directory configuration option we should just generate pg_xlog directory based on it. But
# while we have a separate pg_xlog directory, and while we have a default value for it, we'll still
# base it on pg_data_directory if it was set.
if not data_dir:
data_dir = "/var/lib/pgsql/data"
site_config.setdefault("pg_xlog_directory",
os.path.join(data_dir, "pg_xlog"))
obj_store = site_config["object_storage"] or {}
if not obj_store:
pass
elif "storage_type" not in obj_store:
raise InvalidConfigurationError(
"Site {!r}: storage_type not defined for object_storage".
format(site_name))
elif obj_store["storage_type"] == "local" and obj_store.get(
"directory") == config.get("backup_location"):
raise InvalidConfigurationError(
"Site {!r}: invalid 'local' target directory {!r}, must be different from 'backup_location'"
.format(site_name, config.get("backup_location")))
else:
try:
get_class_for_transfer(obj_store["storage_type"])
except ImportError as ex:
raise InvalidConfigurationError(
"Site {0!r} object_storage: {1.__class__.__name__!s}: {1!s}"
.format(site_name, ex))
# Set command paths and check their versions per site. We use a configured value if one was provided
# (either at top level or per site), if it wasn't provided but we have a valid pg_data_directory with
# PG_VERSION in it we'll look for commands for that version from the expected paths for Debian and
# RHEL/Fedora PGDG packages or otherwise fall back to iterating over the available versions.
# Instead of setting paths explicitly for both commands, it's also possible to just set the
# pg_bin_directory to point to the version-specific bin directory.
bin_dir = site_config.get("pg_bin_directory")
for command in ["pg_basebackup", "pg_receivexlog"]:
command_key = "{}_path".format(command)
command_path = site_config.get(command_key) or config.get(
command_key)
if not command_path:
command_path = os.path.join(bin_dir,
command) if bin_dir else None
if not command_path or not os.path.exists(command_path):
pg_version = site_config["pg_data_directory_version"]
command_path = find_pg_binary(
command, [pg_version] if pg_version else None)
site_config[command_key] = command_path
if check_commands and site_config["active"]:
if not command_path or not os.path.exists(command_path):
raise InvalidConfigurationError(
"Site {!r} command {!r} not found".format(
site_name, command))
version_output = subprocess.check_output(
[command_path, "--version"])
version_string = version_output.decode("ascii").strip()
site_config[command +
"_version"] = convert_pg_command_version_to_number(
version_string)
else:
site_config[command + "_version"] = None
return config
def set_and_check_config_defaults(config,
*,
check_commands=True,
check_pgdata=True):
# TODO: consider implementing a real configuration schema at some point
# misc global defaults
config.setdefault("backup_location", None)
config.setdefault("http_address", PGHOARD_HOST)
config.setdefault("http_port", PGHOARD_PORT)
config.setdefault("alert_file_dir",
config.get("backup_location") or os.getcwd())
config.setdefault("json_state_file_path",
"/var/lib/pghoard/pghoard_state.json")
config.setdefault("maintenance_mode_file",
"/var/lib/pghoard/maintenance_mode_file")
config.setdefault("log_level", "INFO")
config.setdefault("path_prefix",
"") # deprecated, used in the default path for sites
config.setdefault("upload_retries_warning_limit", 3)
# default to cpu_count + 1 compression threads
config.setdefault("compression", {}).setdefault(
"thread_count",
get_cpu_count() + 1,
)
# default to cpu_count + 3 transfer threads (max 20)
config.setdefault("transfer", {}).setdefault(
"thread_count",
min(get_cpu_count() + 3, 20),
)
# default to prefetching transfer.thread_count objects
config.setdefault("restore_prefetch", config["transfer"]["thread_count"])
# if compression algorithm is not explicitly set prefer snappy if it's available
if snappy is not None:
config["compression"].setdefault("algorithm", "snappy")
else:
config["compression"].setdefault("algorithm", "lzma")
config["compression"].setdefault("level", 0)
# defaults for sites
config.setdefault("backup_sites", {})
for site_name, site_config in config["backup_sites"].items():
site_config.setdefault("active", True)
site_config.setdefault("active_backup_mode", "pg_receivexlog")
site_config.setdefault("basebackup_chunk_size", 1024 * 1024 * 1024 * 2)
site_config.setdefault("basebackup_chunks_in_progress", 5)
site_config.setdefault("basebackup_count", 2)
site_config.setdefault("basebackup_interval_hours", 24)
# NOTE: stream_compression removed from documentation after 1.6.0 release
site_config.setdefault(
"basebackup_mode",
"pipe" if site_config.get("stream_compression") else "basic")
site_config.setdefault("encryption_key_id", None)
site_config.setdefault("object_storage", None)
site_config.setdefault("prefix",
os.path.join(config["path_prefix"], site_name))
# NOTE: pg_data_directory doesn't have a default value
data_dir = site_config.get("pg_data_directory")
if not data_dir and check_pgdata:
raise InvalidConfigurationError(
"Site {!r}: pg_data_directory must be set".format(site_name))
if check_pgdata:
version_file = os.path.join(data_dir,
"PG_VERSION") if data_dir else None
with open(version_file, "r") as fp:
site_config["pg_data_directory_version"] = fp.read().strip()
obj_store = site_config["object_storage"] or {}
if not obj_store:
pass
elif "storage_type" not in obj_store:
raise InvalidConfigurationError(
"Site {!r}: storage_type not defined for object_storage".
format(site_name))
elif obj_store["storage_type"] == "local" and obj_store.get(
"directory") == config.get("backup_location"):
raise InvalidConfigurationError(
"Site {!r}: invalid 'local' target directory {!r}, must be different from 'backup_location'"
.format(site_name, config.get("backup_location")))
else:
try:
get_class_for_transfer(obj_store)
except ImportError as ex:
raise InvalidConfigurationError(
"Site {0!r} object_storage: {1.__class__.__name__!s}: {1!s}"
.format(site_name, ex))
# Set command paths and check their versions per site. We use a configured value if one was provided
# (either at top level or per site), if it wasn't provided but we have a valid pg_data_directory with
# PG_VERSION in it we'll look for commands for that version from the expected paths for Debian and
# RHEL/Fedora PGDG packages or otherwise fall back to iterating over the available versions.
# Instead of setting paths explicitly for both commands, it's also possible to just set the
# pg_bin_directory to point to the version-specific bin directory.
bin_dir = site_config.get("pg_bin_directory")
for command in ["pg_basebackup", "pg_receivexlog"]:
# NOTE: pg_basebackup_path and pg_receivexlog_path removed from documentation after 1.6.0 release
command_key = "{}_path".format(command)
command_path = site_config.get(command_key) or config.get(
command_key)
if not command_path:
command_path = os.path.join(bin_dir,
command) if bin_dir else None
if not command_path or not os.path.exists(command_path):
pg_versions_to_check = None
if "pg_data_directory_version" in site_config:
pg_versions_to_check = [
site_config["pg_data_directory_version"]
]
command_path, _ = find_pg_binary(command,
pg_versions_to_check)
site_config[command_key] = command_path
if check_commands and site_config["active"]:
if not command_path or not os.path.exists(command_path):
raise InvalidConfigurationError(
"Site {!r} command {!r} not found from path {}".format(
site_name, command, command_path))
version_output = subprocess.check_output(
[command_path, "--version"])
version_string = version_output.decode("ascii").strip()
site_config[command +
"_version"] = convert_pg_command_version_to_number(
version_string)
else:
site_config[command + "_version"] = None
return config
def set_config_defaults(config, *, check_commands=True):
# TODO: consider implementing a real configuration schema at some point
# misc global defaults
config.setdefault("backup_location", None)
config.setdefault("http_address", PGHOARD_HOST)
config.setdefault("http_port", PGHOARD_PORT)
config.setdefault("alert_file_dir",
config.get("backup_location") or os.getcwd())
config.setdefault("json_state_file_path",
"/tmp/pghoard_state.json") # XXX: get a better default
config.setdefault("log_level", "INFO")
config.setdefault("path_prefix", "")
config.setdefault("upload_retries_warning_limit", 3)
# set command paths and check their versions
for command in ["pg_basebackup", "pg_receivexlog"]:
command_path = config.setdefault(command + "_path",
"/usr/bin/" + command)
if check_commands:
version_output = subprocess.check_output(
[command_path, "--version"])
version_string = version_output.decode("ascii").strip()
config[command +
"_version"] = convert_pg_command_version_to_number(
version_string)
else:
config[command + "_version"] = None
# default to 5 compression and transfer threads
config.setdefault("compression", {}).setdefault("thread_count", 5)
config.setdefault("transfer", {}).setdefault("thread_count", 5)
# default to prefetching min(#compressors, #transferagents) - 1 objects so all
# operations where prefetching is used run fully in parallel without waiting to start
config.setdefault(
"restore_prefetch",
min(config["compression"]["thread_count"],
config["transfer"]["thread_count"]) - 1)
# if compression algorithm is not explicitly set prefer snappy if it's available
if snappy is not None:
config["compression"].setdefault("algorithm", "snappy")
else:
config["compression"].setdefault("algorithm", "lzma")
# defaults for sites
config.setdefault("backup_sites", {})
for site_config in config["backup_sites"].values():
site_config.setdefault("active", True)
site_config.setdefault("active_backup_mode", "pg_receivexlog")
site_config.setdefault("basebackup_count", 2)
site_config.setdefault("basebackup_interval_hours", 24)
site_config.setdefault("encryption_key_id", None)
site_config.setdefault("object_storage", None)
site_config.setdefault("pg_xlog_directory",
"/var/lib/pgsql/data/pg_xlog")
site_config.setdefault("stream_compression", False)
obj_store = site_config["object_storage"] or {}
if obj_store.get("type") == "local" and obj_store.get(
"directory") == config.get("backup_location"):
raise InvalidConfigurationError(
"Invalid 'local' target directory {!r}, must be different from 'backup_location'"
.format(config.get("backup_location")))
return config
