def test_archive_sync(self, db, pghoard):
log = logging.getLogger("test_archive_sync")
store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
def list_archive(folder):
if folder == "timeline":
matcher = wal.TIMELINE_RE.match
else:
matcher = wal.XLOG_RE.match
path_to_list = "{}/{}".format(pghoard.test_site, folder)
files_found, files_total = 0, 0
for obj in store.list_path(path_to_list):
fname = os.path.basename(obj["name"])
files_total += 1
if matcher(fname):
files_found += 1
yield fname
log.info("Listed %r, %r out of %r matched %r pattern", path_to_list, files_found, files_total, folder)
# create a basebackup to start with
self._run_and_wait_basebackup(pghoard, db, "pipe")
# force a couple of wal segment switches
start_xlog, _ = self._switch_xlog(db, 4)
# we should have at least 4 xlog files now (there may be more in
# case other tests created them -- we share a single postresql
# cluster between all tests)
pg_xlog_dir = pghoard.config["backup_sites"][pghoard.test_site]["pg_xlog_directory"]
pg_xlogs = {f for f in os.listdir(pg_xlog_dir) if wal.XLOG_RE.match(f) and f > start_xlog}
assert len(pg_xlogs) >= 4
# create a couple of "recycled" xlog files that we must ignore
last_xlog = sorted(pg_xlogs)[-1]
dummy_data = b"x" * (16 * 2 ** 20)
def write_dummy_xlog(inc):
filename = "{:024X}".format((int(last_xlog, 16) + inc))
print("writing dummy xlog file", filename)
open(os.path.join(pg_xlog_dir, filename), "wb").write(dummy_data)
return filename
recycled1 = write_dummy_xlog(1)
recycled2 = write_dummy_xlog(2)
# check what we have archived, there should be at least the three
# above xlogs that are NOT there at the moment
archived_xlogs = set(list_archive("xlog"))
assert len(pg_xlogs - archived_xlogs) >= 4
# now perform an archive sync
arsy = ArchiveSync()
arsy.run(["--site", pghoard.test_site, "--config", pghoard.config_path])
# and now archive should include all our xlogs
archived_xlogs = set(list_archive("xlog"))
# the recycled files must not appear in archived files
assert recycled1 not in archived_xlogs
assert recycled2 not in archived_xlogs
# the regular wals must be archived
assert archived_xlogs.issuperset(pg_xlogs)
# if we delete a wal file that's not the latest archival it should
# get synced to the archive as we don't have a basebackup newer than
# it
current_wal = arsy.get_current_wal_file()
old_xlogs = sorted(wal for wal in pg_xlogs if wal < current_wal)
store.delete_key(os.path.join(pghoard.test_site, "xlog", old_xlogs[-2]))
arsy.run(["--site", pghoard.test_site, "--config", pghoard.config_path])
archived_xlogs = set(list_archive("xlog"))
assert archived_xlogs.issuperset(pg_xlogs)
# delete the topmost wal file, this should cause resync too
store.delete_key(os.path.join(pghoard.test_site, "xlog", old_xlogs[-1]))
arsy.run(["--site", pghoard.test_site, "--config", pghoard.config_path])
archived_xlogs = set(list_archive("xlog"))
assert archived_xlogs.issuperset(pg_xlogs)
# let's do a little dance to turn our DB into a standby and then
# promote it, forcing a timeline switch
db.kill(force=False)
with open(os.path.join(db.pgdata, "recovery.conf"), "w") as fp:
fp.write(
"standby_mode = 'on'\n"
"recovery_target_timeline = 'latest'\n"
"restore_command = 'false'\n"
)
# start PG and promote it
db.run_pg()
db.run_cmd("pg_ctl", "-D", db.pgdata, "promote")
time.sleep(5) # TODO: instead of sleeping, poll the db until ready
# we should have a single timeline file in pg_xlog now
pg_xlog_timelines = {f for f in os.listdir(pg_xlog_dir) if wal.TIMELINE_RE.match(f)}
assert len(pg_xlog_timelines) > 0
# but there should be nothing archived as archive_command wasn't setup
archived_timelines = set(list_archive("timeline"))
assert len(archived_timelines) == 0
# let's hit archive sync
arsy.run(["--site", pghoard.test_site, "--config", pghoard.config_path])
# now we should have an archived timeline
archived_timelines = set(list_archive("timeline"))
assert archived_timelines.issuperset(pg_xlog_timelines)
assert "00000002.history" in archived_timelines
# let's take a new basebackup
self._run_and_wait_basebackup(pghoard, db, "basic")
# nuke archives and resync them
for name in list_archive(folder="timeline"):
store.delete_key(os.path.join(pghoard.test_site, "timeline", name))
for name in list_archive(folder="xlog"):
store.delete_key(os.path.join(pghoard.test_site, "xlog", name))
self._switch_xlog(db, 1)
arsy.run(["--site", pghoard.test_site, "--config", pghoard.config_path])
archived_xlogs = set(list_archive("xlog"))
# assume the same timeline file as before and one to three wal files
assert len(archived_xlogs) >= 1
assert len(archived_xlogs) <= 3
archived_timelines = set(list_archive("timeline"))
assert list(archived_timelines) == ["00000002.history"]
def test_archive_sync(self, db, pghoard):
log = logging.getLogger("test_archive_sync")
store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
def list_archive(folder):
if folder == "timeline":
matcher = wal.TIMELINE_RE.match
else:
matcher = wal.WAL_RE.match
path_to_list = "{}/{}".format(pghoard.test_site, folder)
files_found, files_total = 0, 0
for obj in store.list_path(path_to_list):
fname = os.path.basename(obj["name"])
files_total += 1
if matcher(fname):
files_found += 1
yield fname
log.info("Listed %r, %r out of %r matched %r pattern", path_to_list, files_found, files_total, folder)
# create a basebackup to start with
self._run_and_wait_basebackup(pghoard, db, "pipe")
# force a couple of wal segment switches
start_wal, _ = self._switch_wal(db, 4)
# we should have at least 4 WAL files now (there may be more in
# case other tests created them -- we share a single postresql
# cluster between all tests)
pg_wal_dir = get_pg_wal_directory(pghoard.config["backup_sites"][pghoard.test_site])
pg_wals = {f for f in os.listdir(pg_wal_dir) if wal.WAL_RE.match(f) and f > start_wal}
assert len(pg_wals) >= 4
# create a couple of "recycled" xlog files that we must ignore
last_wal = sorted(pg_wals)[-1]
dummy_data = b"x" * (16 * 2 ** 20)
def write_dummy_wal(inc):
filename = "{:024X}".format((int(last_wal, 16) + inc))
print("Writing dummy WAL file", filename)
open(os.path.join(pg_wal_dir, filename), "wb").write(dummy_data)
return filename
recycled1 = write_dummy_wal(1)
recycled2 = write_dummy_wal(2)
# check what we have archived, there should be at least the three
# above WALs that are NOT there at the moment
archived_wals = set(list_archive("xlog"))
assert len(pg_wals - archived_wals) >= 4
# now perform an archive sync
arsy = ArchiveSync()
arsy.run(["--site", pghoard.test_site, "--config", pghoard.config_path])
# and now archive should include all our WALs
archived_wals = set(list_archive("xlog"))
# the recycled files must not appear in archived files
assert recycled1 not in archived_wals
assert recycled2 not in archived_wals
# the regular wals must be archived
assert archived_wals.issuperset(pg_wals)
# if we delete a wal file that's not the latest archival it should
# get synced to the archive as we don't have a basebackup newer than
# it
current_wal = arsy.get_current_wal_file()
old_wals = sorted(wal for wal in pg_wals if wal < current_wal)
store.delete_key(os.path.join(pghoard.test_site, "xlog", old_wals[-2]))
arsy.run(["--site", pghoard.test_site, "--config", pghoard.config_path])
archived_wals = set(list_archive("xlog"))
assert archived_wals.issuperset(pg_wals)
# delete the topmost wal file, this should cause resync too
store.delete_key(os.path.join(pghoard.test_site, "xlog", old_wals[-1]))
arsy.run(["--site", pghoard.test_site, "--config", pghoard.config_path])
archived_wals = set(list_archive("xlog"))
assert archived_wals.issuperset(pg_wals)
# let's do a little dance to turn our DB into a standby and then
# promote it, forcing a timeline switch
db.kill(force=False)
with open(os.path.join(db.pgdata, "recovery.conf"), "w") as fp:
fp.write(
"standby_mode = 'on'\n"
"recovery_target_timeline = 'latest'\n"
"restore_command = 'false'\n"
)
# start PG and promote it
db.run_pg()
db.run_cmd("pg_ctl", "-D", db.pgdata, "promote")
time.sleep(5) # TODO: instead of sleeping, poll the db until ready
# we should have a single timeline file in pg_xlog/pg_wal now
pg_wal_timelines = {f for f in os.listdir(pg_wal_dir) if wal.TIMELINE_RE.match(f)}
assert len(pg_wal_timelines) > 0
# but there should be nothing archived as archive_command wasn't setup
archived_timelines = set(list_archive("timeline"))
assert len(archived_timelines) == 0
# let's hit archive sync
arsy.run(["--site", pghoard.test_site, "--config", pghoard.config_path])
# now we should have an archived timeline
archived_timelines = set(list_archive("timeline"))
assert archived_timelines.issuperset(pg_wal_timelines)
assert "00000002.history" in archived_timelines
# let's take a new basebackup
self._run_and_wait_basebackup(pghoard, db, "basic")
# nuke archives and resync them
for name in list_archive(folder="timeline"):
store.delete_key(os.path.join(pghoard.test_site, "timeline", name))
for name in list_archive(folder="xlog"):
store.delete_key(os.path.join(pghoard.test_site, "xlog", name))
self._switch_wal(db, 1)
arsy.run(["--site", pghoard.test_site, "--config", pghoard.config_path])
archived_wals = set(list_archive("xlog"))
# assume the same timeline file as before and one to three wal files
assert len(archived_wals) >= 1
assert len(archived_wals) <= 3
archived_timelines = set(list_archive("timeline"))
assert list(archived_timelines) == ["00000002.history"]
