def replication_resume(params):
if not postgresql.is_secondary():
hookenv.action_fail("Not a hot standby")
return
con = postgresql.connect()
con.autocommit = True
offset = postgresql.wal_received_offset(con)
hookenv.action_set(dict(offset=offset))
cur = con.cursor()
if postgresql.has_version("10"):
cur.execute("SELECT pg_is_wal_replay_paused()")
else:
cur.execute("SELECT pg_is_xlog_replay_paused()")
if cur.fetchone()[0] is False:
# Not a failure, per lp:1670613
hookenv.action_set(dict(result="Already resumed"))
return
if postgresql.has_version("10"):
cur.execute("SELECT pg_wal_replay_resume()")
else:
cur.execute("SELECT pg_xlog_replay_resume()")
hookenv.action_set(dict(result="Resumed"))
def test_has_version(self, version):
version.return_value = '9.4'
self.assertTrue(postgresql.has_version('9.1'))
self.assertTrue(postgresql.has_version('9.4'))
self.assertFalse(postgresql.has_version('9.5'))
# New version scheme starting PostgreSQL 10
version.return_value = '10'
self.assertTrue(postgresql.has_version('9.6'))
self.assertFalse(postgresql.has_version('11'))
def postgresql_conf_defaults():
"""Return the postgresql.conf defaults, which we parse from config.yaml"""
# We load defaults from the extra_pg_conf default in config.yaml,
# which ensures that they never get out of sync.
raw = helpers.config_yaml()["options"]["extra_pg_conf"]["default"]
defaults = postgresql.parse_config(raw)
# And recalculate some defaults, which could get out of sync.
# Settings with mandatory minimums like wal_senders are handled
# later, in ensure_viable_postgresql_conf().
ram = int(host.get_total_ram() / (1024 * 1024)) # Working in megabytes.
# Default shared_buffers to 25% of ram, minimum 16MB, maximum 8GB,
# per current best practice rules of thumb. Rest is cache.
shared_buffers = max(min(math.ceil(ram * 0.25), 8192), 16)
effective_cache_size = max(1, ram - shared_buffers)
defaults["shared_buffers"] = "{} MB".format(shared_buffers)
defaults["effective_cache_size"] = "{} MB".format(effective_cache_size)
# PostgreSQL 10 introduces multiple password encryption methods.
if postgresql.has_version("10"):
# Change this to scram-sha-256 next LTS release, when we can
# start assuming clients have libpq 10. The setting can of
# course still be overridden in the config.
defaults["password_encryption"] = "md5"
else:
defaults["password_encryption"] = True
return defaults
def failover():
if get_following() is None:
hookenv.log("Failover already in progress", DEBUG)
return
# Stop replicating the doomed master, or we risk diverging
# timelines.
pg12 = postgresql.has_version("12")
if pg12:
path = postgresql.hot_standby_conf_path()
template = "hot_standby.conf.tmpl"
else:
path = postgresql.recovery_conf_path()
template = "recovery.conf.tmpl"
templating.render(template,
path, {},
owner="postgres",
group="postgres",
perms=0o600)
if pg12:
touch(postgresql.hot_standby_signal_path())
# Kick off a rolling restart to apply the change.
reactive.set_state("postgresql.cluster.needs_restart")
# Publish the change after the restart.
set_following(None)
reactive.set_state("postgresql.replication.publish_following")
def update_recovery_conf(follow):
assert follow != hookenv.local_unit()
peer_rel = helpers.get_peer_relation()
follow_relinfo = peer_rel.get(follow)
assert follow_relinfo is not None, "Invalid upstream {}".format(follow)
current_follow = get_following()
if follow != current_follow:
status_set("maintenance", "Following new unit {}".format(follow))
set_following(follow)
# Setting the state to defer publication until after restart.
reactive.set_state("postgresql.replication.publish_following")
else:
# Even though the master is unchanged, we still regenerate
# recovery.conf in case connection details such as IP addresses
# have changed.
hookenv.log("Continuing to follow {}".format(follow))
pg12 = postgresql.has_version("12")
if pg12:
path = postgresql.hot_standby_conf_path()
template = "hot_standby.conf.tmpl"
else:
path = postgresql.recovery_conf_path()
template = "recovery.conf.tmpl"
config = hookenv.config()
data = dict(
streaming_replication=config["streaming_replication"],
host=follow_relinfo["host"],
port=follow_relinfo["port"],
user=replication_username(),
password=leader_get("replication_password"),
)
if reactive.helpers.is_state("postgresql.wal_e.enabled"):
data["restore_command"] = wal_e.wal_e_restore_command()
templating.render(template,
path,
data,
owner="postgres",
group="postgres",
perms=0o600)
if pg12:
touch(postgresql.hot_standby_signal_path())
# Use @when_file_changed for this when Issue #44 is resolved.
if reactive.helpers.any_file_changed([path]):
reactive.set_state("postgresql.cluster.needs_restart")
if reactive.is_state("postgresql.replication.cloned"):
reactive.set_state("postgresql.replication.check_following")
def update_postgresql_conf():
settings = assemble_postgresql_conf()
path = postgresql.postgresql_conf_path()
with open(path, "r") as f:
pg_conf = f.read()
start_mark = "### BEGIN JUJU SETTINGS ###"
end_mark = "### END JUJU SETTINGS ###"
# Strip the existing settings section, including the markers.
pg_conf = re.sub(
r"^\s*{}.*^\s*{}\s*$".format(re.escape(start_mark), re.escape(end_mark)),
"",
pg_conf,
flags=re.I | re.M | re.DOTALL,
)
for k in settings:
# Comment out conflicting options. We could just allow later
# options to override earlier ones, but this is less surprising.
pg_conf = re.sub(
r"^\s*({}[\s=].*)$".format(re.escape(k)),
r"# juju # \1",
pg_conf,
flags=re.M | re.I,
)
# Store the updated charm options. This is compared with the
# live config to detect if a restart is required.
store = unitdata.kv()
current_prefix = "postgresql.cluster.pgconf.current."
store.unsetrange(prefix=current_prefix)
store.update(settings, prefix=current_prefix)
# Generate the charm config section, adding it to the end of the
# config file.
simple_re = re.compile(r"^[-.\w]+$")
override_section = [start_mark]
for k, v in settings.items():
v = str(v)
assert "\n" not in v, "Invalid config value {!r}".format(v)
if simple_re.search(v) is None:
v = "'{}'".format(v.replace("'", "''"))
override_section.append("{} = {}".format(k, v))
if postgresql.has_version("12"):
override_section.append("include_if_exists '{}'".format(postgresql.hot_standby_conf_path()))
override_section.append(end_mark)
pg_conf += "\n" + "\n".join(override_section)
helpers.rewrite(path, pg_conf)
def promote_to_master():
status_set("maintenance", "Promoting to master")
postgresql.promote()
set_following(None)
publish_following()
while postgresql.is_in_recovery():
status_set("maintenance", "Waiting for startup")
time.sleep(1)
if postgresql.has_version("12"):
assert not os.path.exists(postgresql.hot_standby_signal_path(
)), "standby.signal still exists after promotion"
else:
assert not os.path.exists(postgresql.recovery_conf_path(
)), "recovery.conf still exists after promotion"
update_replication_states()
helpers.ping_peers()
def ensure_viable_postgresql_conf(opts):
def force(**kw):
for k, v in kw.items():
if opts.get(k) != v:
hookenv.log("Setting {} to {}".format(k, v), DEBUG)
opts[k] = v
config = hookenv.config()
rels = context.Relations()
# Number of standby units - count peers and 'master' relations.
num_standbys = len(helpers.get_peer_relation() or {})
for rel in rels["master"].values():
num_standbys += len(rel)
num_clients = 0
for rel in list(rels["db"]) + list(rels["db-admin"]):
num_clients += len(rel)
# Even without replication, replication slots get used by
# pg_basebackup(1). Bump up max_wal_senders so things work. It is
# cheap, so perhaps we should just pump it to several thousand.
min_wal_senders = num_standbys * 2 + 5
if min_wal_senders > int(opts.get("max_wal_senders", 0)):
force(max_wal_senders=min_wal_senders)
# We used to calculate a minimum max_connections here, ensuring
# that we had at least one per client and enough for replication
# and backups. It wasn't much use though, as the major variable
# is not the number of clients but how many connections the
# clients open (connection pools of 20 or more are not uncommon).
# lp:1594667 required the calculation to be synchronized, or just
# removed. So removed to avoid complexity for dubious gains.
#
# max_connections. One per client unit, plus replication.
# max_wal_senders = int(opts.get('max_wal_senders', 0))
# assert max_wal_senders > 0
# min_max_connections = max_wal_senders + max(1, num_clients)
#
min_max_connections = 100
if min_max_connections > int(opts.get("max_connections", 0)):
force(max_connections=min_max_connections)
# We want 'hot_standby' at a minimum, as it lets us run
# pg_basebackup() and it is recommended over the more
# minimal 'archive'. Is it worth only enabling the higher-still
# 'logical' level only when necessary? How do we detect that?
force(hot_standby=True)
if postgresql.has_version("9.4"):
force(wal_level="logical")
else:
force(wal_level="hot_standby")
# Having two config options for the one setting is confusing. Perhaps
# we should deprecate this.
if num_standbys and (int(config["replicated_wal_keep_segments"]) > int(opts.get("wal_keep_segments", 0))):
force(wal_keep_segments=config["replicated_wal_keep_segments"])
# Log shipping with WAL-E.
if config["wal_e_storage_uri"]:
force(archive_mode="on") # Boolean pre-9.5, enum 9.5+
force(archive_command=wal_e.wal_e_archive_command())
# Log destinations for syslog. This charm only supports standard
# Debian logging, or Debian + syslog. This will grow more complex in
# the future, as the local logs are redundant if you are using syslog
# for log aggregation, and we will want to add csvlog because it is
# so much easier to parse.
if context.Relations()["syslog"]:
force(
log_destination="stderr,syslog",
syslog_ident=hookenv.local_unit().replace("/", "_"),
)
def wal_e_restore():
reactive.remove_state("action.wal-e-restore")
params = hookenv.action_get()
backup = params["backup-name"].strip().replace("-", "_")
storage_uri = params["storage-uri"].strip()
ship_uri = hookenv.config().get("wal_e_storage_uri")
if storage_uri == ship_uri:
hookenv.action_fail(
"The storage-uri parameter is identical to "
"the wal_e_storage_uri config setting. Your "
"restoration source cannot be the same as the "
"folder you are archiving too to avoid corrupting "
"the backups."
)
return
if not params["confirm"]:
m = "Recovery from {}.".format(storage_uri)
if ship_uri:
m += "\nContents of {} will be destroyed.".format(ship_uri)
m += "\nExisting local database will be destroyed."
m += "\nRerun action with 'confirm=true' to proceed."
hookenv.action_set({"info": m})
return
with tempfile.TemporaryDirectory(prefix="wal-e", suffix="envdir") as envdir:
update_wal_e_env_dir(envdir, storage_uri)
# Confirm there is a backup to restore
backups = wal_e_list_backups(envdir)
if not backups:
hookenv.action_fail("No backups found at {}".format(storage_uri))
return
if backup != "LATEST" and backup not in (b["name"] for b in backups):
hookenv.action_fail("Backup {} not found".format(backup))
return
# Shutdown PostgreSQL. Note we want this action to run synchronously,
# so there is no opportunity to ask permission from the leader. If
# there are other units cloning this database, those clone operations
# will fail. Which seems preferable to blocking a recovery operation
# in any case, because if we are doing disaster recovery we generally
# want to do it right now.
status_set("maintenance", "Stopping PostgreSQL for backup restoration")
postgresql.stop()
# Trash the existing database. Its dangerous to do this first, but
# we probably need the space.
data_dir = postgresql.data_dir() # May be a symlink
for content in os.listdir(data_dir):
cpath = os.path.join(data_dir, content)
if os.path.isdir(cpath) and not os.path.islink(cpath):
shutil.rmtree(cpath)
else:
os.remove(cpath)
# WAL-E recover
status_set("maintenance", "Restoring backup {}".format(backup))
wal_e_run(["backup-fetch", data_dir, backup], envdir=envdir)
# Create recovery.conf to complete recovery
is_master = reactive.is_state("postgresql.replication.is_master")
standby_mode = "off" if is_master else "on"
if params.get("target-time"):
target_time = "recovery_target_time='{}'" "".format(params["target-time"])
else:
target_time = ""
target_action = "promote" if is_master else "shutdown"
immediate = "" if is_master else "recovery_target='immediate'"
helpers.write(
postgresql.recovery_conf_path(),
dedent(
"""\
# Managed by Juju. PITR in progress.
standby_mode = {}
restore_command='{}'
recovery_target_timeline = {}
recovery_target_action = {}
{}
{}
"""
).format(
standby_mode,
wal_e_restore_command(envdir=envdir),
params["target-timeline"],
target_action,
target_time,
immediate,
),
mode=0o600,
user="******",
group="postgres",
)
# Avoid circular import. We could also avoid the import entirely
# with a sufficiently complex set of handlers in the replication
# module, but that seems to be a worse solution. Better to break
# out this action into a separate module.
from reactive.postgresql import replication
if is_master:
if ship_uri:
# If master, trash the configured wal-e storage. This may
# contain WAL and backups from the old cluster which will
# conflict with the new cluster. Hopefully it does not
# contain anything important, because we have no way to
# prompt the user for confirmation.
wal_e_run(["delete", "--confirm", "everything"])
# Then, wait for recovery and promotion.
postgresql.start()
con = postgresql.connect()
cur = con.cursor()
while True:
if postgresql.has_version("10"):
cur.execute(
"""SELECT pg_is_in_recovery(),
pg_last_wal_replay_lsn()"""
)
else:
cur.execute(
"""SELECT pg_is_in_recovery(),
pg_last_xlog_replay_location()"""
)
in_rec, loc = cur.fetchone()
if not in_rec:
break
status_set("maintenance", "Recovery at {}".format(loc))
time.sleep(10)
else:
# If standby, startup and wait for recovery to complete and
# shutdown.
status_set("maintenance", "Recovery")
# Startup might shutdown immediately and look like a failure.
postgresql.start(ignore_failure=True)
# No recovery point status yet for standbys, as we would need
# to handle connection failures when the DB shuts down. We
# should do this.
while postgresql.is_running():
time.sleep(5)
replication.update_recovery_conf(follow=replication.get_master())
# Reactive handlers will deal with the rest of the cleanup.
# eg. ensuring required users and roles exist
replication.update_replication_states()
reactive.remove_state("postgresql.cluster.configured")
reactive.toggle_state("postgresql.cluster.is_running", postgresql.is_running())
reactive.remove_state("postgresql.nagios.user_ensured")
reactive.remove_state("postgresql.replication.replication_user_created")
reactive.remove_state("postgresql.client.published")
def test_has_version(self, version):
version.return_value = "9.4"
self.assertTrue(postgresql.has_version("9.1"))
self.assertTrue(postgresql.has_version("9.4"))
self.assertFalse(postgresql.has_version("9.5"))
def ensure_viable_postgresql_conf(opts):
def force(**kw):
for k, v in kw.items():
if opts.get(k) != v:
hookenv.log('Setting {} to {}'.format(k, v), DEBUG)
opts[k] = v
config = hookenv.config()
rels = context.Relations()
# Number of standby units - count peers and 'master' relations.
num_standbys = len(rels.peer or {})
for rel in rels['master'].values():
num_standbys += len(rel)
num_clients = 0
for rel in list(rels['db']) + list(rels['db-admin']):
num_clients += len(rel)
# Even without replication, replication slots get used by
# pg_basebackup(1). Bump up max_wal_senders so things work. It is
# cheap, so perhaps we should just pump it to several thousand.
min_wal_senders = num_standbys * 2 + 5
if min_wal_senders > int(opts.get('max_wal_senders', 0)):
force(max_wal_senders=min_wal_senders)
# We used to calculate a minimum max_connections here, ensuring
# that we had at least one per client and enough for replication
# and backups. It wasn't much use though, as the major variable
# is not the number of clients but how many connections the
# clients open (connection pools of 20 or more are not uncommon).
# lp:1594667 required the calculation to be synchronized, or just
# removed. So removed to avoid complexity for dubious gains.
#
# max_connections. One per client unit, plus replication.
# max_wal_senders = int(opts.get('max_wal_senders', 0))
# assert max_wal_senders > 0
# min_max_connections = max_wal_senders + max(1, num_clients)
#
min_max_connections = 100
if min_max_connections > int(opts.get('max_connections', 0)):
force(max_connections=min_max_connections)
# We want 'hot_standby' at a minimum, as it lets us run
# pg_basebackup() and it is recommended over the more
# minimal 'archive'. Is it worth only enabling the higher-still
# 'logical' level only when necessary? How do we detect that?
force(hot_standby=True)
if postgresql.has_version('9.4'):
force(wal_level='logical')
else:
force(wal_level='hot_standby')
# Having two config options for the one setting is confusing. Perhaps
# we should deprecate this.
if num_standbys and (int(config['replicated_wal_keep_segments']) >
int(opts.get('wal_keep_segments', 0))):
force(wal_keep_segments=config['replicated_wal_keep_segments'])
# Log shipping with WAL-E.
if config['wal_e_storage_uri']:
force(archive_mode='on') # Boolean pre-9.5, enum 9.5+
force(archive_command=wal_e.wal_e_archive_command())
# Log destinations for syslog. This charm only supports standard
# Debian logging, or Debian + syslog. This will grow more complex in
# the future, as the local logs are redundant if you are using syslog
# for log aggregation, and we will want to add csvlog because it is
# so much easier to parse.
if context.Relations()['syslog']:
force(log_destination='stderr,syslog',
syslog_ident=hookenv.local_unit().replace('/', '_'))
def clone_master():
master = get_master()
peer_rel = helpers.get_peer_relation()
master_relinfo = peer_rel[master]
# Be paranoid since we are about to destroy data.
assert not reactive.helpers.is_state("postgresql.replication.is_master")
assert not reactive.helpers.is_state("postgresql.cluster.is_running")
# We use realpath on data_dir as it may have been replaced with
# a symbolic link, so we empty and recreate the actual directory
# and the links remain in place.
data_dir = os.path.realpath(postgresql.data_dir())
if os.path.exists(data_dir):
hookenv.log("Removing {} in preparation for clone".format(data_dir))
shutil.rmtree(data_dir)
helpers.makedirs(data_dir, mode=0o700, user="******", group="postgres")
if postgresql.has_version("10"):
wal_method = "--wal-method=stream"
else:
wal_method = "--xlog-method=stream"
cmd = [
"sudo",
"-H", # -H needed to locate $HOME/.pgpass
"-u",
"postgres",
"pg_basebackup",
"-D",
postgresql.data_dir(),
"-h",
master_relinfo["host"],
"-p",
master_relinfo["port"],
"--checkpoint=fast",
"--progress",
wal_method,
"--no-password",
"--username=_juju_repl",
]
hookenv.log("Cloning {} with {}".format(master, " ".join(cmd)))
status_set("maintenance", "Cloning {}".format(master))
try:
# Switch to a directory the postgres user can access.
with helpers.switch_cwd("/tmp"):
subprocess.check_call(cmd, universal_newlines=True)
except subprocess.CalledProcessError as x:
hookenv.log("Clone failed with {}".format(x), ERROR)
# We failed, and the local cluster is broken.
status_set("blocked", "Failed to clone {}".format(master))
postgresql.drop_cluster()
reactive.remove_state("postgresql.cluster.configured")
reactive.remove_state("postgresql.cluster.created")
# Terminate. We need this hook to exit, rather than enter a loop.
raise SystemExit(0)
update_recovery_conf(follow=master)
reactive.set_state("postgresql.replication.cloned")
update_replication_states()
