def setup_method(self, method):
super().setup_method(method)
self.config = self.config_template({
"backup_sites": {
self.test_site: {
"basebackup_count": 1,
"basebackup_interval_hours": 1,
"nodes": [
{
"host": "127.0.0.4",
},
],
},
},
})
config_path = os.path.join(self.temp_dir, "pghoard.json")
write_json_file(config_path, self.config)
self.pghoard = PGHoard(config_path)
# This is the "final storage location" when using "local" storage type
self.local_storage_dir = os.path.join(
self.config["backup_sites"][self.test_site]["object_storage"]
["directory"], self.test_site)
self.real_check_pg_server_version = self.pghoard.check_pg_server_version
self.pghoard.check_pg_server_version = Mock(return_value=90404)
self.real_check_pg_versions_ok = self.pghoard.check_pg_versions_ok
self.pghoard.check_pg_versions_ok = Mock(return_value=True)
def setup_method(self, method):
super().setup_method(method)
self.config = self.config_template()
self.config["backup_sites"][self.test_site].update({
"basebackup_count":
1,
"basebackup_interval_hours":
1,
"nodes": [{
"host": "127.0.0.4"
}],
})
config_path = os.path.join(self.temp_dir, "pghoard.json")
write_json_file(config_path, self.config)
os.makedirs(self.config["alert_file_dir"], exist_ok=True)
backup_site_path = os.path.join(self.config["backup_location"],
self.test_site)
self.compressed_xlog_path = os.path.join(backup_site_path, "xlog")
os.makedirs(self.compressed_xlog_path)
self.basebackup_path = os.path.join(backup_site_path, "basebackup")
os.makedirs(self.basebackup_path)
self.pghoard = PGHoard(config_path)
self.real_check_pg_server_version = self.pghoard.check_pg_server_version
self.pghoard.check_pg_server_version = Mock(return_value=90404)
self.real_check_pg_versions_ok = self.pghoard.check_pg_versions_ok
self.pghoard.check_pg_versions_ok = Mock(return_value=True)
def pghoard(db, tmpdir, request): # pylint: disable=redefined-outer-name
test_site = request.function.__name__
if os.environ.get("pghoard_test_walreceiver"):
active_backup_mode = "walreceiver"
else:
active_backup_mode = "pg_receivexlog"
config = {
"alert_file_dir": os.path.join(str(tmpdir), "alerts"),
"backup_location": os.path.join(str(tmpdir), "backupspool"),
"backup_sites": {
test_site: {
"active_backup_mode": active_backup_mode,
"basebackup_count": 2,
"basebackup_interval_hours": 24,
"pg_bin_directory": db.pgbin,
"pg_data_directory": db.pgdata,
"nodes": [db.user],
"object_storage": {
"storage_type": "local",
"directory": os.path.join(str(tmpdir), "backups"),
},
},
},
"compression": {
"algorithm": "snappy" if snappy else "lzma",
},
"http_address": "127.0.0.1",
"http_port": random.randint(1024, 32000),
"json_state_file_path": tmpdir.join("pghoard_state.json").strpath,
"maintenance_mode_file": tmpdir.join("maintenance_mode_file").strpath,
}
confpath = os.path.join(str(tmpdir), "config.json")
with open(confpath, "w") as fp:
json.dump(config, fp)
backup_site_path = os.path.join(config["backup_location"], test_site)
basebackup_path = os.path.join(backup_site_path, "basebackup")
backup_xlog_path = os.path.join(backup_site_path, "xlog")
backup_timeline_path = os.path.join(backup_site_path, "timeline")
os.makedirs(config["alert_file_dir"])
os.makedirs(basebackup_path)
os.makedirs(backup_xlog_path)
os.makedirs(backup_timeline_path)
pgh = PGHoard(confpath)
pgh.test_site = test_site
pgh.start_threads_on_startup()
if snappy:
pgh.Compressor = snappy.StreamCompressor
else:
pgh.Compressor = lambda: lzma.LZMACompressor(preset=0) # pylint: disable=redefined-variable-type
time.sleep(0.05) # Hack to give the server time to start up
yield pgh
pgh.quit()
def setup_method(self, method):
super().setup_method(method)
self.config = self.config_template({
"backup_sites": {
self.test_site: {
"basebackup_count": 1,
"basebackup_interval_hours": 1,
"nodes": [
{
"host": "127.0.0.4",
},
],
},
},
})
config_path = os.path.join(self.temp_dir, "pghoard.json")
write_json_file(config_path, self.config)
self.pghoard = PGHoard(config_path)
# This is the "final storage location" when using "local" storage type
self.local_storage_dir = os.path.join(self.config["backup_sites"][self.test_site]["object_storage"]["directory"],
self.test_site)
self.real_check_pg_server_version = self.pghoard.check_pg_server_version
self.pghoard.check_pg_server_version = Mock(return_value=90404)
self.real_check_pg_versions_ok = self.pghoard.check_pg_versions_ok
self.pghoard.check_pg_versions_ok = Mock(return_value=True)
def setUp(self):
self.temp_dir = tempfile.mkdtemp()
config_path = os.path.join(self.temp_dir, "pghoard.json")
create_json_conf(config_path, self.temp_dir)
self.xlog_path = os.path.join(self.temp_dir, "default", "xlog")
os.makedirs(self.xlog_path)
self.pghoard = PGHoard(config_path)
def pghoard(db, tmpdir, request): # pylint: disable=redefined-outer-name
test_site = request.function.__name__
if os.environ.get("pghoard_test_walreceiver"):
active_backup_mode = "walreceiver"
else:
active_backup_mode = "pg_receivexlog"
config = {
"alert_file_dir": os.path.join(str(tmpdir), "alerts"),
"backup_location": os.path.join(str(tmpdir), "backupspool"),
"backup_sites": {
test_site: {
"active_backup_mode": active_backup_mode,
"basebackup_count": 2,
"basebackup_interval_hours": 24,
"pg_bin_directory": db.pgbin,
"pg_data_directory": db.pgdata,
"pg_xlog_directory": os.path.join(db.pgdata, "pg_xlog"),
"nodes": [db.user],
"object_storage": {
"storage_type": "local",
"directory": os.path.join(str(tmpdir), "backups"),
},
},
},
"http_address": "127.0.0.1",
"http_port": random.randint(1024, 32000),
"compression": {
"algorithm": "snappy" if snappy else "lzma",
}
}
confpath = os.path.join(str(tmpdir), "config.json")
with open(confpath, "w") as fp:
json.dump(config, fp)
backup_site_path = os.path.join(config["backup_location"], test_site)
basebackup_path = os.path.join(backup_site_path, "basebackup")
backup_xlog_path = os.path.join(backup_site_path, "xlog")
backup_timeline_path = os.path.join(backup_site_path, "timeline")
os.makedirs(config["alert_file_dir"])
os.makedirs(basebackup_path)
os.makedirs(backup_xlog_path)
os.makedirs(backup_timeline_path)
pgh = PGHoard(confpath)
pgh.test_site = test_site
pgh.start_threads_on_startup()
if snappy:
pgh.Compressor = snappy.StreamCompressor
else:
pgh.Compressor = lambda: lzma.LZMACompressor(preset=0) # pylint: disable=redefined-variable-type
time.sleep(0.05) # Hack to give the server time to start up
yield pgh
pgh.quit()
class TestPGHoard(TestCase):
def setUp(self):
self.temp_dir = tempfile.mkdtemp()
config_path = os.path.join(self.temp_dir, "pghoard.json")
create_json_conf(config_path, self.temp_dir)
self.xlog_path = os.path.join(self.temp_dir, "default", "xlog")
os.makedirs(self.xlog_path)
self.pghoard = PGHoard(config_path)
def test_alert_files(self):
alert_file_path = os.path.join(self.temp_dir, "test_alert")
self.pghoard.create_alert_file("test_alert")
self.assertTrue(os.path.exists(alert_file_path))
self.pghoard.delete_alert_file("test_alert")
self.assertFalse(os.path.exists(alert_file_path))
def test_backup_state_file(self):
self.pghoard.write_backup_state_to_json_file()
state_path = os.path.join(self.temp_dir, "state.json")
with open(state_path, "r") as fp:
state = json.load(fp)
empty_state = {
"startup_time": self.pghoard.state["startup_time"],
"backup_sites": {},
"compressors": [{}, {}],
"queues": {
"compression_queue": 0,
"transfer_queue": 0,
},
"data_transfer": {},
"transfer_agents": [{}, {}],
"pg_receivexlogs": {},
"pg_basebackups": {},
}
assert empty_state == state
def test_startup_walk_for_missed_files(self):
with open(os.path.join(self.xlog_path, "000000010000000000000004"), "wb") as fp:
fp.write(b"foo")
self.pghoard.startup_walk_for_missed_files()
self.assertEqual(self.pghoard.compression_queue.qsize(), 1)
def tearDown(self):
self.pghoard.quit()
shutil.rmtree(self.temp_dir)
def setup_method(self, method):
super().setup_method(method)
config_path = os.path.join(self.temp_dir, "pghoard.json")
self.config = create_json_conf(config_path, self.temp_dir, self.test_site)
backup_site_path = os.path.join(self.config["backup_location"], self.test_site)
self.compressed_xlog_path = os.path.join(backup_site_path, "xlog")
os.makedirs(self.compressed_xlog_path)
self.basebackup_path = os.path.join(backup_site_path, "basebackup")
os.makedirs(self.basebackup_path)
self.pghoard = PGHoard(config_path)
self.real_check_pg_server_version = self.pghoard.check_pg_server_version
self.pghoard.check_pg_server_version = Mock(return_value="psql (PostgreSQL) 9.4.4")
self.real_check_pg_versions_ok = self.pghoard.check_pg_versions_ok
self.pghoard.check_pg_versions_ok = Mock(return_value=True)
def setup_method(self, method):
super().setup_method(method)
self.config = self.config_template()
self.config["backup_sites"][self.test_site].update({
"basebackup_count": 1,
"basebackup_interval_hours": 1,
"nodes": [{"host": "127.0.0.4"}],
})
config_path = os.path.join(self.temp_dir, "pghoard.json")
write_json_file(config_path, self.config)
os.makedirs(self.config["alert_file_dir"], exist_ok=True)
backup_site_path = os.path.join(self.config["backup_location"], self.test_site)
self.compressed_xlog_path = os.path.join(backup_site_path, "xlog")
os.makedirs(self.compressed_xlog_path)
self.basebackup_path = os.path.join(backup_site_path, "basebackup")
os.makedirs(self.basebackup_path)
self.pghoard = PGHoard(config_path)
self.real_check_pg_server_version = self.pghoard.check_pg_server_version
self.pghoard.check_pg_server_version = Mock(return_value=90404)
self.real_check_pg_versions_ok = self.pghoard.check_pg_versions_ok
self.pghoard.check_pg_versions_ok = Mock(return_value=True)
class TestPGHoard(PGHoardTestCase):
def setup_method(self, method):
super().setup_method(method)
self.config = self.config_template()
self.config["backup_sites"][self.test_site].update({
"basebackup_count": 1,
"basebackup_interval_hours": 1,
"nodes": [{"host": "127.0.0.4"}],
})
config_path = os.path.join(self.temp_dir, "pghoard.json")
write_json_file(config_path, self.config)
compat.makedirs(self.config["alert_file_dir"], exist_ok=True)
self.pghoard = PGHoard(config_path)
# This is the "final storage location" when using "local" storage type
self.local_storage_dir = os.path.join(self.config["backup_sites"][self.test_site]["object_storage"]["directory"],
self.test_site)
self.real_check_pg_server_version = self.pghoard.check_pg_server_version
self.pghoard.check_pg_server_version = Mock(return_value=90404)
self.real_check_pg_versions_ok = self.pghoard.check_pg_versions_ok
self.pghoard.check_pg_versions_ok = Mock(return_value=True)
def teardown_method(self, method):
self.pghoard.quit()
self.pghoard.check_pg_server_version = self.real_check_pg_server_version
self.pghoard.check_pg_versions_ok = self.real_check_pg_versions_ok
super().teardown_method(method)
@patch("subprocess.check_output")
def test_handle_site(self, subprocess_mock):
subprocess_mock.return_value = b"""\
systemid|6222667313856416063
timeline|1
xlogpos|0/B003760
dbname|"""
self.pghoard.handle_site(self.test_site, self.config["backup_sites"][self.test_site])
assert len(self.pghoard.receivexlogs) == 1
assert len(self.pghoard.time_of_last_backup_check) == 1
def test_get_local_basebackups_info(self):
basebackup_storage_path = os.path.join(self.local_storage_dir, "basebackup")
os.makedirs(basebackup_storage_path)
assert self.pghoard.get_remote_basebackups_info(self.test_site) == []
bb_path = os.path.join(basebackup_storage_path, "2015-07-03_0")
# Handle case where metadata file does not exist
assert self.pghoard.get_remote_basebackups_info(self.test_site) == []
metadata_file_path = bb_path + ".metadata"
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(metadata_file_path, "w") as fp:
json.dump({"start-time": "2015-07-03 12:00:00+00:00"}, fp)
available_backup = self.pghoard.get_remote_basebackups_info(self.test_site)[0]
assert available_backup["name"] == "2015-07-03_0"
start_time = datetime.datetime(2015, 7, 3, 12, tzinfo=datetime.timezone.utc)
assert available_backup["metadata"] == {"start-time": start_time}
bb_path = os.path.join(basebackup_storage_path, "2015-07-02_9")
metadata_file_path = bb_path + ".metadata"
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(metadata_file_path, "w") as fp:
json.dump({"start-time": "2015-07-02 12:00:00+00:00"}, fp)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert basebackups[0]["name"] == "2015-07-02_9"
assert basebackups[1]["name"] == "2015-07-03_0"
bb_path = os.path.join(basebackup_storage_path, "2015-07-02_10")
metadata_file_path = bb_path + ".metadata"
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(metadata_file_path, "w") as fp:
json.dump({"start-time": "2015-07-02 22:00:00+00"}, fp)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert basebackups[0]["name"] == "2015-07-02_9"
assert basebackups[1]["name"] == "2015-07-02_10"
assert basebackups[2]["name"] == "2015-07-03_0"
def test_local_check_backup_count_and_state(self):
basebackup_storage_path = os.path.join(self.local_storage_dir, "basebackup")
xlog_storage_path = os.path.join(self.local_storage_dir, "xlog")
os.makedirs(basebackup_storage_path)
os.makedirs(xlog_storage_path)
self.pghoard.set_state_defaults(self.test_site)
assert self.pghoard.get_remote_basebackups_info(self.test_site) == []
def write_backup_and_wal_files(what):
for bb, wals in what.items():
if bb:
bb_path = os.path.join(basebackup_storage_path, bb)
date_parts = [int(part) for part in bb.replace("_", "-").split("-")]
start_time = datetime.datetime(*date_parts, tzinfo=datetime.timezone.utc)
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(bb_path + ".metadata", "w") as fp:
json.dump({
"start-wal-segment": wals[0],
"start-time": start_time.isoformat(),
}, fp)
for wal in wals:
with open(os.path.join(xlog_storage_path, wal), "wb") as fp:
fp.write(b"something")
backups_and_wals = {
"2015-08-25_0": [
# NOTE: gap between this and next segment means that cleanup shouldn't find this
"000000010000000A000000FB",
],
"2015-08-25_1": [
"000000020000000A000000FD",
"000000020000000A000000FE",
],
"2015-08-25_2": [
"000000030000000A000000FF",
"000000030000000B00000000",
"000000030000000B00000001",
"000000040000000B00000002",
],
"2015-08-25_3": [
# Both of these should be saved
"000000040000000B00000003",
"000000040000000B00000004",
],
}
write_backup_and_wal_files(backups_and_wals)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 4
self.pghoard.check_backup_count_and_state(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
assert len(os.listdir(xlog_storage_path)) == 3
# Put all WAL segments between 1 and 9 in place to see that they're deleted and we don't try to go back
# any further from TLI 1. Note that timeline 3 is now "empty" so deletion shouldn't touch timelines 2
# or 1.
new_backups_and_wals = {
"": [
"000000020000000A000000FC",
"000000020000000A000000FD",
"000000020000000A000000FE",
"000000020000000A000000FF",
"000000020000000B00000000",
"000000020000000B00000001",
"000000020000000B00000002",
],
"2015-08-25_4": [
"000000040000000B00000005",
],
}
write_backup_and_wal_files(new_backups_and_wals)
assert len(os.listdir(xlog_storage_path)) == 11
self.pghoard.check_backup_count_and_state(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
expected_wal_count = len(backups_and_wals["2015-08-25_0"])
expected_wal_count += len(new_backups_and_wals[""])
expected_wal_count += len(new_backups_and_wals["2015-08-25_4"])
assert len(os.listdir(xlog_storage_path)) == expected_wal_count
# Now put WAL files in place with no gaps anywhere
gapless_backups_and_wals = {
"2015-08-25_3": [
"000000030000000B00000003",
"000000040000000B00000004",
],
"2015-08-25_4": [
"000000040000000B00000005",
],
}
write_backup_and_wal_files(gapless_backups_and_wals)
assert len(os.listdir(xlog_storage_path)) >= 10
self.pghoard.check_backup_count_and_state(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
assert len(os.listdir(xlog_storage_path)) == 1
def test_alert_files(self):
alert_file_path = os.path.join(self.config["alert_file_dir"], "test_alert")
create_alert_file(self.pghoard.config, "test_alert")
assert os.path.exists(alert_file_path) is True
delete_alert_file(self.pghoard.config, "test_alert")
assert os.path.exists(alert_file_path) is False
def test_backup_state_file(self):
self.pghoard.write_backup_state_to_json_file()
state_path = self.config["json_state_file_path"]
thread_count = 5
with open(state_path, "r") as fp:
state = json.load(fp)
empty_state = {
"startup_time": self.pghoard.state["startup_time"],
"backup_sites": {},
"compressors": [{}] * thread_count,
"queues": {
"compression_queue": 0,
"transfer_queue": 0,
},
"transfer_agents": [{}] * thread_count,
"pg_receivexlogs": {},
"pg_basebackups": {},
}
assert empty_state == state
def test_startup_walk_for_missed_compressed_files(self):
compressed_xlog_path, _ = self.pghoard.create_backup_site_paths(self.test_site)
with open(os.path.join(compressed_xlog_path, "000000010000000000000004"), "wb") as fp:
fp.write(b"foo")
with open(os.path.join(compressed_xlog_path, "000000010000000000000004.metadata"), "wb") as fp:
fp.write(b"{}")
with open(os.path.join(compressed_xlog_path, "0000000F.history"), "wb") as fp:
fp.write(b"foo")
with open(os.path.join(compressed_xlog_path, "0000000F.history.metadata"), "wb") as fp:
fp.write(b"{}")
with open(os.path.join(compressed_xlog_path, "000000010000000000000004xyz"), "wb") as fp:
fp.write(b"foo")
with open(os.path.join(compressed_xlog_path, "000000010000000000000004xyz.metadata"), "wb") as fp:
fp.write(b"{}")
self.pghoard.startup_walk_for_missed_files()
assert self.pghoard.compression_queue.qsize() == 0
assert self.pghoard.transfer_queue.qsize() == 2
def test_startup_walk_for_missed_uncompressed_files(self):
compressed_xlog_path, _ = self.pghoard.create_backup_site_paths(self.test_site)
uncompressed_xlog_path = compressed_xlog_path + "_incoming"
with open(os.path.join(uncompressed_xlog_path, "000000010000000000000004"), "wb") as fp:
fp.write(b"foo")
with open(os.path.join(uncompressed_xlog_path, "00000002.history"), "wb") as fp:
fp.write(b"foo")
with open(os.path.join(uncompressed_xlog_path, "000000010000000000000004xyz"), "wb") as fp:
fp.write(b"foo")
self.pghoard.startup_walk_for_missed_files()
assert self.pghoard.compression_queue.qsize() == 2
assert self.pghoard.transfer_queue.qsize() == 0
def pghoard_base(
db,
tmpdir,
request,
compression="snappy", # pylint: disable=redefined-outer-name
transfer_count=None,
metrics_cfg=None):
test_site = request.function.__name__
if os.environ.get("pghoard_test_walreceiver"):
active_backup_mode = "walreceiver"
else:
active_backup_mode = "pg_receivexlog"
if compression == "snappy" and not snappy:
compression = "lzma"
config = {
"alert_file_dir": os.path.join(str(tmpdir), "alerts"),
"backup_location": os.path.join(str(tmpdir), "backupspool"),
"backup_sites": {
test_site: {
"active_backup_mode": active_backup_mode,
"basebackup_count": 2,
"basebackup_interval_hours": 24,
"pg_bin_directory": db.pgbin,
"pg_data_directory": db.pgdata,
"nodes": [db.user],
"object_storage": {
"storage_type": "local",
"directory": os.path.join(str(tmpdir), "backups"),
},
},
},
"compression": {
"algorithm": compression,
},
"http_address": "127.0.0.1",
"http_port": random.randint(1024, 32000),
"json_state_file_path": tmpdir.join("pghoard_state.json").strpath,
"maintenance_mode_file": tmpdir.join("maintenance_mode_file").strpath,
# Set process count to 1 to avoid launching subprocesses during basebackup tests.
# The new processes would be created with fork, which doesn't work properly due to
# all the fds and other things that are created during typical test setup. There
# is separate test case that executes the multiprocess version.
"restore_process_count": 1,
"tar_executable": "tar",
}
if metrics_cfg is not None:
config.update(metrics_cfg)
if transfer_count is not None:
config["transfer"] = {"thread_count": transfer_count}
confpath = os.path.join(str(tmpdir), "config.json")
with open(confpath, "w") as fp:
json.dump(config, fp)
backup_site_path = os.path.join(config["backup_location"], test_site)
basebackup_path = os.path.join(backup_site_path, "basebackup")
backup_xlog_path = os.path.join(backup_site_path, "xlog")
backup_timeline_path = os.path.join(backup_site_path, "timeline")
os.makedirs(config["alert_file_dir"])
os.makedirs(basebackup_path)
os.makedirs(backup_xlog_path)
os.makedirs(backup_timeline_path)
pgh = PGHoard(confpath)
pgh.test_site = test_site
pgh.start_threads_on_startup()
if compression == "snappy":
pgh.Compressor = snappy.StreamCompressor
else:
pgh.Compressor = lambda: lzma.LZMACompressor(preset=0) # pylint: disable=redefined-variable-type
time.sleep(0.05) # Hack to give the server time to start up
yield pgh
pgh.quit()
class TestPGHoard(PGHoardTestCase):
def setup_method(self, method):
super().setup_method(method)
self.config = self.config_template({
"backup_sites": {
self.test_site: {
"basebackup_count": 1,
"basebackup_interval_hours": 1,
"nodes": [
{
"host": "127.0.0.4",
},
],
},
},
})
config_path = os.path.join(self.temp_dir, "pghoard.json")
write_json_file(config_path, self.config)
self.pghoard = PGHoard(config_path)
# This is the "final storage location" when using "local" storage type
self.local_storage_dir = os.path.join(
self.config["backup_sites"][self.test_site]["object_storage"]
["directory"], self.test_site)
self.real_check_pg_server_version = self.pghoard.check_pg_server_version
self.pghoard.check_pg_server_version = Mock(return_value=90404)
self.real_check_pg_versions_ok = self.pghoard.check_pg_versions_ok
self.pghoard.check_pg_versions_ok = Mock(return_value=True)
def teardown_method(self, method):
self.pghoard.quit()
self.pghoard.check_pg_server_version = self.real_check_pg_server_version
self.pghoard.check_pg_versions_ok = self.real_check_pg_versions_ok
super().teardown_method(method)
@patch("subprocess.check_output")
def test_handle_site(self, subprocess_mock):
subprocess_mock.return_value = b"""\
systemid|6222667313856416063
timeline|1
xlogpos|0/B003760
dbname|"""
self.pghoard.handle_site(self.test_site,
self.config["backup_sites"][self.test_site])
assert len(self.pghoard.receivexlogs) == 1 or len(
self.pghoard.walreceivers) == 1
assert len(self.pghoard.time_of_last_backup_check) == 1
def test_get_local_basebackups_info(self):
basebackup_storage_path = os.path.join(self.local_storage_dir,
"basebackup")
os.makedirs(basebackup_storage_path)
assert self.pghoard.get_remote_basebackups_info(self.test_site) == []
bb_path = os.path.join(basebackup_storage_path, "2015-07-03_0")
# Handle case where metadata file does not exist
assert self.pghoard.get_remote_basebackups_info(self.test_site) == []
metadata_file_path = bb_path + ".metadata"
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(metadata_file_path, "w") as fp:
json.dump({"start-time": "2015-07-03 12:00:00+00:00"}, fp)
available_backup = self.pghoard.get_remote_basebackups_info(
self.test_site)[0]
assert available_backup["name"] == "2015-07-03_0"
start_time = datetime.datetime(2015,
7,
3,
12,
tzinfo=datetime.timezone.utc)
assert available_backup["metadata"]["start-time"] == start_time
assert available_backup["metadata"]["backup-reason"] == "scheduled"
assert available_backup["metadata"]["normalized-backup-time"] is None
assert available_backup["metadata"]["backup-decision-time"]
bb_path = os.path.join(basebackup_storage_path, "2015-07-02_9")
metadata_file_path = bb_path + ".metadata"
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(metadata_file_path, "w") as fp:
json.dump({"start-time": "2015-07-02 12:00:00+00:00"}, fp)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert basebackups[0]["name"] == "2015-07-02_9"
assert basebackups[1]["name"] == "2015-07-03_0"
bb_path = os.path.join(basebackup_storage_path, "2015-07-02_10")
metadata_file_path = bb_path + ".metadata"
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(metadata_file_path, "w") as fp:
json.dump({"start-time": "2015-07-02 22:00:00+00"}, fp)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert basebackups[0]["name"] == "2015-07-02_9"
assert basebackups[1]["name"] == "2015-07-02_10"
assert basebackups[2]["name"] == "2015-07-03_0"
def test_determine_backups_to_delete(self):
now = datetime.datetime.now(datetime.timezone.utc)
bbs = [
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=10, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=9, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=9, hours=1)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=8, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=7, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=6, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=6, hours=20)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=5, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=4, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=3, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=2, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=1, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(hours=4)
}
},
]
site_config = {
"basebackup_count": 4,
"basebackup_count_min": 2,
"basebackup_interval_hours": 24,
}
bbs_copy = list(bbs)
to_delete = self.pghoard.determine_backups_to_delete(
basebackups=bbs_copy, site_config=site_config)
assert len(bbs_copy) == 4
assert len(to_delete) == len(bbs) - len(bbs_copy)
assert to_delete == bbs[:len(to_delete)]
assert bbs_copy == bbs[len(to_delete):]
site_config["basebackup_count"] = 16
site_config["basebackup_age_days_max"] = 8
bbs_copy = list(bbs)
to_delete = self.pghoard.determine_backups_to_delete(
basebackups=bbs_copy, site_config=site_config)
# 3 of the backups are too old (start time + interval is over 8 days in the past)
assert len(bbs_copy) == 10
assert len(to_delete) == len(bbs) - len(bbs_copy)
assert to_delete == bbs[:len(to_delete)]
assert bbs_copy == bbs[len(to_delete):]
site_config["basebackup_count"] = 9
bbs_copy = list(bbs)
to_delete = self.pghoard.determine_backups_to_delete(
basebackups=bbs_copy, site_config=site_config)
# basebackup_count trumps backup age and backups are removed even though they're not too old
assert len(bbs_copy) == 9
assert len(to_delete) == len(bbs) - len(bbs_copy)
assert to_delete == bbs[:len(to_delete)]
assert bbs_copy == bbs[len(to_delete):]
site_config["basebackup_count"] = 16
site_config["basebackup_age_days_max"] = 2
site_config["basebackup_count_min"] = 6
bbs_copy = list(bbs)
to_delete = self.pghoard.determine_backups_to_delete(
basebackups=bbs_copy, site_config=site_config)
# basebackup_count_min ensures not that many backups are removed even though they're too old
assert len(bbs_copy) == 6
assert len(to_delete) == len(bbs) - len(bbs_copy)
assert to_delete == bbs[:len(to_delete)]
assert bbs_copy == bbs[len(to_delete):]
site_config["basebackup_count_min"] = 2
bbs_copy = list(bbs)
to_delete = self.pghoard.determine_backups_to_delete(
basebackups=bbs_copy, site_config=site_config)
# 3 of the backups are new enough (start time less than 3 days in the past)
assert len(bbs_copy) == 3
assert len(to_delete) == len(bbs) - len(bbs_copy)
assert to_delete == bbs[:len(to_delete)]
assert bbs_copy == bbs[len(to_delete):]
def test_local_refresh_backup_list_and_delete_old(self):
basebackup_storage_path = os.path.join(self.local_storage_dir,
"basebackup")
wal_storage_path = os.path.join(self.local_storage_dir, "xlog")
os.makedirs(basebackup_storage_path)
os.makedirs(wal_storage_path)
self.pghoard.set_state_defaults(self.test_site)
assert self.pghoard.get_remote_basebackups_info(self.test_site) == []
def write_backup_and_wal_files(what):
for bb, wals in what.items():
if bb:
bb_path = os.path.join(basebackup_storage_path, bb)
date_parts = [
int(part) for part in bb.replace("_", "-").split("-")
]
start_time = datetime.datetime(
*date_parts, tzinfo=datetime.timezone.utc)
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(bb_path + ".metadata", "w") as fp:
json.dump(
{
"start-wal-segment": wals[0],
"start-time": start_time.isoformat(),
}, fp)
for wal in wals:
with open(os.path.join(wal_storage_path, wal), "wb") as fp:
fp.write(b"something")
backups_and_wals = {
"2015-08-25_0": [
# NOTE: gap between this and next segment means that cleanup shouldn't find this
"000000010000000A000000FB",
],
"2015-08-25_1": [
"000000020000000A000000FD",
"000000020000000A000000FE",
],
"2015-08-25_2": [
"000000030000000A000000FF",
"000000030000000B00000000",
"000000030000000B00000001",
"000000040000000B00000002",
],
"2015-08-25_3": [
# Both of these should be saved
"000000040000000B00000003",
"000000040000000B00000004",
],
}
write_backup_and_wal_files(backups_and_wals)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 4
self.pghoard.refresh_backup_list_and_delete_old(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
assert len(os.listdir(wal_storage_path)) == 3
# Put all WAL segments between 1 and 9 in place to see that they're deleted and we don't try to go back
# any further from TLI 1. Note that timeline 3 is now "empty" so deletion shouldn't touch timelines 2
# or 1.
new_backups_and_wals = {
"": [
"000000020000000A000000FC",
"000000020000000A000000FD",
"000000020000000A000000FE",
"000000020000000A000000FF",
"000000020000000B00000000",
"000000020000000B00000001",
"000000020000000B00000002",
],
"2015-08-25_4": [
"000000040000000B00000005",
],
}
write_backup_and_wal_files(new_backups_and_wals)
assert len(os.listdir(wal_storage_path)) == 11
self.pghoard.refresh_backup_list_and_delete_old(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
expected_wal_count = len(backups_and_wals["2015-08-25_0"])
expected_wal_count += len(new_backups_and_wals[""])
expected_wal_count += len(new_backups_and_wals["2015-08-25_4"])
assert len(os.listdir(wal_storage_path)) == expected_wal_count
# Now put WAL files in place with no gaps anywhere
gapless_backups_and_wals = {
"2015-08-25_3": [
"000000030000000B00000003",
"000000040000000B00000004",
],
"2015-08-25_4": [
"000000040000000B00000005",
],
}
write_backup_and_wal_files(gapless_backups_and_wals)
assert len(os.listdir(wal_storage_path)) >= 10
self.pghoard.refresh_backup_list_and_delete_old(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
assert len(os.listdir(wal_storage_path)) == 1
def test_alert_files(self):
alert_file_path = os.path.join(self.config["alert_file_dir"],
"test_alert")
create_alert_file(self.pghoard.config, "test_alert")
assert os.path.exists(alert_file_path) is True
delete_alert_file(self.pghoard.config, "test_alert")
assert os.path.exists(alert_file_path) is False
def test_backup_state_file(self):
self.pghoard.write_backup_state_to_json_file()
state_path = self.config["json_state_file_path"]
with open(state_path, "r") as fp:
state = json.load(fp)
empty_state = {
"startup_time": self.pghoard.state["startup_time"],
"backup_sites": {},
"compressors": [{}] * self.config["compression"]["thread_count"],
"queues": {
"compression_queue": 0,
"transfer_queue": 0,
},
"transfer_agents": [{}] * self.config["transfer"]["thread_count"],
"pg_receivexlogs": {},
"pg_basebackups": {},
"walreceivers": {},
}
assert empty_state == state
def test_startup_walk_for_missed_compressed_files(self):
compressed_wal_path, _ = self.pghoard.create_backup_site_paths(
self.test_site)
with open(
os.path.join(compressed_wal_path, "000000010000000000000004"),
"wb") as fp:
fp.write(b"foo")
with open(
os.path.join(compressed_wal_path,
"000000010000000000000004.metadata"), "wb") as fp:
fp.write(b"{}")
with open(os.path.join(compressed_wal_path, "0000000F.history"),
"wb") as fp:
fp.write(b"foo")
with open(
os.path.join(compressed_wal_path, "0000000F.history.metadata"),
"wb") as fp:
fp.write(b"{}")
with open(
os.path.join(compressed_wal_path,
"000000010000000000000004xyz"), "wb") as fp:
fp.write(b"foo")
with open(
os.path.join(compressed_wal_path,
"000000010000000000000004xyz.metadata"),
"wb") as fp:
fp.write(b"{}")
self.pghoard.startup_walk_for_missed_files()
assert self.pghoard.compression_queue.qsize() == 0
assert self.pghoard.transfer_queue.qsize() == 2
def test_startup_walk_for_missed_uncompressed_files(self):
compressed_wal_path, _ = self.pghoard.create_backup_site_paths(
self.test_site)
uncompressed_wal_path = compressed_wal_path + "_incoming"
with open(
os.path.join(uncompressed_wal_path,
"000000010000000000000004"), "wb") as fp:
fp.write(b"foo")
with open(os.path.join(uncompressed_wal_path, "00000002.history"),
"wb") as fp:
fp.write(b"foo")
with open(
os.path.join(uncompressed_wal_path,
"000000010000000000000004xyz"), "wb") as fp:
fp.write(b"foo")
self.pghoard.startup_walk_for_missed_files()
assert self.pghoard.compression_queue.qsize() == 2
assert self.pghoard.transfer_queue.qsize() == 0
class TestPGHoard(PGHoardTestCase):
def setup_method(self, method):
super().setup_method(method)
self.config = self.config_template()
self.config["backup_sites"][self.test_site].update({
"basebackup_count":
1,
"basebackup_interval_hours":
1,
"nodes": [{
"host": "127.0.0.4"
}],
})
config_path = os.path.join(self.temp_dir, "pghoard.json")
write_json_file(config_path, self.config)
os.makedirs(self.config["alert_file_dir"], exist_ok=True)
backup_site_path = os.path.join(self.config["backup_location"],
self.test_site)
self.compressed_xlog_path = os.path.join(backup_site_path, "xlog")
os.makedirs(self.compressed_xlog_path)
self.basebackup_path = os.path.join(backup_site_path, "basebackup")
os.makedirs(self.basebackup_path)
self.pghoard = PGHoard(config_path)
self.real_check_pg_server_version = self.pghoard.check_pg_server_version
self.pghoard.check_pg_server_version = Mock(return_value=90404)
self.real_check_pg_versions_ok = self.pghoard.check_pg_versions_ok
self.pghoard.check_pg_versions_ok = Mock(return_value=True)
def teardown_method(self, method):
self.pghoard.quit()
self.pghoard.check_pg_server_version = self.real_check_pg_server_version
self.pghoard.check_pg_versions_ok = self.real_check_pg_versions_ok
super().teardown_method(method)
@patch("subprocess.check_output")
def test_handle_site(self, subprocess_mock):
subprocess_mock.return_value = b"""\
systemid|6222667313856416063
timeline|1
xlogpos|0/B003760
dbname|"""
self.pghoard.handle_site(self.test_site,
self.config["backup_sites"][self.test_site])
assert len(self.pghoard.receivexlogs) == 1
assert len(self.pghoard.time_of_last_backup_check) == 1
def test_get_local_basebackups_info(self):
assert self.pghoard.get_remote_basebackups_info(self.test_site) == []
bb_path = os.path.join(self.basebackup_path, "2015-07-03_0")
# Handle case where metadata file does not exist
assert self.pghoard.get_remote_basebackups_info(self.test_site) == []
metadata_file_path = bb_path + ".metadata"
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(metadata_file_path, "w") as fp:
json.dump({"start-time": "2015-07-03 12:00:00+00:00"}, fp)
available_backup = self.pghoard.get_remote_basebackups_info(
self.test_site)[0]
assert available_backup["name"] == "2015-07-03_0"
start_time = datetime.datetime(2015,
7,
3,
12,
tzinfo=datetime.timezone.utc)
assert available_backup["metadata"] == {"start-time": start_time}
bb_path = os.path.join(self.basebackup_path, "2015-07-02_9")
metadata_file_path = bb_path + ".metadata"
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(metadata_file_path, "w") as fp:
json.dump({"start-time": "2015-07-02 12:00:00+00:00"}, fp)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert basebackups[0]["name"] == "2015-07-02_9"
assert basebackups[1]["name"] == "2015-07-03_0"
bb_path = os.path.join(self.basebackup_path, "2015-07-02_10")
metadata_file_path = bb_path + ".metadata"
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(metadata_file_path, "w") as fp:
json.dump({"start-time": "2015-07-02 22:00:00+00"}, fp)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert basebackups[0]["name"] == "2015-07-02_9"
assert basebackups[1]["name"] == "2015-07-02_10"
assert basebackups[2]["name"] == "2015-07-03_0"
def test_local_check_backup_count_and_state(self):
self.pghoard.set_state_defaults(self.test_site)
assert self.pghoard.get_remote_basebackups_info(self.test_site) == []
def write_backup_and_wal_files(what):
for bb, wals in what.items():
if bb:
bb_path = os.path.join(self.basebackup_path, bb)
date_parts = [
int(part) for part in bb.replace("_", "-").split("-")
]
start_time = datetime.datetime(
*date_parts, tzinfo=datetime.timezone.utc)
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(bb_path + ".metadata", "w") as fp:
json.dump(
{
"start-wal-segment": wals[0],
"start-time": start_time.isoformat(),
}, fp)
for wal in wals:
with open(os.path.join(self.compressed_xlog_path, wal),
"wb") as fp:
fp.write(b"something")
backups_and_wals = {
"2015-08-25_0": [
# NOTE: gap between this and next segment means that cleanup shouldn't find this
"000000010000000A000000FB",
],
"2015-08-25_1": [
"000000020000000A000000FD",
"000000020000000A000000FE",
],
"2015-08-25_2": [
"000000030000000A000000FF",
"000000030000000B00000000",
"000000030000000B00000001",
"000000040000000B00000002",
],
"2015-08-25_3": [
# Both of these should be saved
"000000040000000B00000003",
"000000040000000B00000004",
],
}
write_backup_and_wal_files(backups_and_wals)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 4
self.pghoard.check_backup_count_and_state(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
assert len(os.listdir(self.compressed_xlog_path)) == 3
# Put all WAL segments between 1 and 9 in place to see that they're deleted and we don't try to go back
# any further from TLI 1. Note that timeline 3 is now "empty" so deletion shouldn't touch timelines 2
# or 1.
new_backups_and_wals = {
"": [
"000000020000000A000000FC",
"000000020000000A000000FD",
"000000020000000A000000FE",
"000000020000000A000000FF",
"000000020000000B00000000",
"000000020000000B00000001",
"000000020000000B00000002",
],
"2015-08-25_4": [
"000000040000000B00000005",
],
}
write_backup_and_wal_files(new_backups_and_wals)
assert len(os.listdir(self.compressed_xlog_path)) == 11
self.pghoard.check_backup_count_and_state(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
expected_wal_count = len(backups_and_wals["2015-08-25_0"])
expected_wal_count += len(new_backups_and_wals[""])
expected_wal_count += len(new_backups_and_wals["2015-08-25_4"])
assert len(os.listdir(self.compressed_xlog_path)) == expected_wal_count
# Now put WAL files in place with no gaps anywhere
gapless_backups_and_wals = {
"2015-08-25_3": [
"000000030000000B00000003",
"000000040000000B00000004",
],
"2015-08-25_4": [
"000000040000000B00000005",
],
}
write_backup_and_wal_files(gapless_backups_and_wals)
assert len(os.listdir(self.compressed_xlog_path)) >= 10
self.pghoard.check_backup_count_and_state(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
assert len(os.listdir(self.compressed_xlog_path)) == 1
def test_alert_files(self):
alert_file_path = os.path.join(self.config["alert_file_dir"],
"test_alert")
create_alert_file(self.pghoard.config, "test_alert")
assert os.path.exists(alert_file_path) is True
delete_alert_file(self.pghoard.config, "test_alert")
assert os.path.exists(alert_file_path) is False
def test_backup_state_file(self):
self.pghoard.write_backup_state_to_json_file()
state_path = self.config["json_state_file_path"]
thread_count = 5
with open(state_path, "r") as fp:
state = json.load(fp)
empty_state = {
"startup_time": self.pghoard.state["startup_time"],
"backup_sites": {},
"compressors": [{}] * thread_count,
"queues": {
"compression_queue": 0,
"transfer_queue": 0,
},
"transfer_agents": [{}] * thread_count,
"pg_receivexlogs": {},
"pg_basebackups": {},
}
assert empty_state == state
def test_startup_walk_for_missed_files(self):
with open(
os.path.join(self.compressed_xlog_path,
"000000010000000000000004"), "wb") as fp:
fp.write(b"foo")
self.pghoard.startup_walk_for_missed_files()
assert self.pghoard.compression_queue.qsize() == 1
class TestPGHoard(PGHoardTestCase):
def setup_method(self, method):
super().setup_method(method)
self.config = self.config_template({
"backup_sites": {
self.test_site: {
"basebackup_count": 1,
"basebackup_interval_hours": 1,
"nodes": [
{
"host": "127.0.0.4",
},
],
},
},
})
config_path = os.path.join(self.temp_dir, "pghoard.json")
write_json_file(config_path, self.config)
self.pghoard = PGHoard(config_path)
# This is the "final storage location" when using "local" storage type
self.local_storage_dir = os.path.join(
self.config["backup_sites"][self.test_site]["object_storage"]
["directory"], self.test_site)
self.real_check_pg_server_version = self.pghoard.check_pg_server_version
self.pghoard.check_pg_server_version = Mock(return_value=90404)
self.real_check_pg_versions_ok = self.pghoard.check_pg_versions_ok
self.pghoard.check_pg_versions_ok = Mock(return_value=True)
def teardown_method(self, method):
self.pghoard.quit()
self.pghoard.check_pg_server_version = self.real_check_pg_server_version
self.pghoard.check_pg_versions_ok = self.real_check_pg_versions_ok
super().teardown_method(method)
@patch("subprocess.check_output")
def test_handle_site(self, subprocess_mock):
subprocess_mock.return_value = b"""\
systemid|6222667313856416063
timeline|1
xlogpos|0/B003760
dbname|"""
self.pghoard.handle_site(self.test_site,
self.config["backup_sites"][self.test_site])
assert len(self.pghoard.receivexlogs) == 1 or len(
self.pghoard.walreceivers) == 1
assert len(self.pghoard.time_of_last_backup_check) == 1
def test_get_local_basebackups_info(self):
basebackup_storage_path = os.path.join(self.local_storage_dir,
"basebackup")
os.makedirs(basebackup_storage_path)
assert self.pghoard.get_remote_basebackups_info(self.test_site) == []
bb_path = os.path.join(basebackup_storage_path, "2015-07-03_0")
# Handle case where metadata file does not exist
assert self.pghoard.get_remote_basebackups_info(self.test_site) == []
metadata_file_path = bb_path + ".metadata"
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(metadata_file_path, "w") as fp:
json.dump({"start-time": "2015-07-03 12:00:00+00:00"}, fp)
available_backup = self.pghoard.get_remote_basebackups_info(
self.test_site)[0]
assert available_backup["name"] == "2015-07-03_0"
start_time = datetime.datetime(2015,
7,
3,
12,
tzinfo=datetime.timezone.utc)
assert available_backup["metadata"]["start-time"] == start_time
assert available_backup["metadata"]["backup-reason"] == "scheduled"
assert available_backup["metadata"]["normalized-backup-time"] is None
assert available_backup["metadata"]["backup-decision-time"]
bb_path = os.path.join(basebackup_storage_path, "2015-07-02_9")
metadata_file_path = bb_path + ".metadata"
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(metadata_file_path, "w") as fp:
json.dump({"start-time": "2015-07-02 12:00:00+00:00"}, fp)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert basebackups[0]["name"] == "2015-07-02_9"
assert basebackups[1]["name"] == "2015-07-03_0"
bb_path = os.path.join(basebackup_storage_path, "2015-07-02_10")
metadata_file_path = bb_path + ".metadata"
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(metadata_file_path, "w") as fp:
json.dump({"start-time": "2015-07-02 22:00:00+00"}, fp)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert basebackups[0]["name"] == "2015-07-02_9"
assert basebackups[1]["name"] == "2015-07-02_10"
assert basebackups[2]["name"] == "2015-07-03_0"
def test_determine_backups_to_delete(self):
now = datetime.datetime.now(datetime.timezone.utc)
bbs = [
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=10, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=9, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=9, hours=1)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=8, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=7, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=6, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=6, hours=20)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=5, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=4, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=3, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=2, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(days=1, hours=4)
}
},
{
"name": "bb1",
"metadata": {
"start-time": now - datetime.timedelta(hours=4)
}
},
]
site_config = {
"basebackup_count": 4,
"basebackup_count_min": 2,
"basebackup_interval_hours": 24,
}
bbs_copy = list(bbs)
to_delete = self.pghoard.determine_backups_to_delete(
basebackups=bbs_copy, site_config=site_config)
assert len(bbs_copy) == 4
assert len(to_delete) == len(bbs) - len(bbs_copy)
assert to_delete == bbs[:len(to_delete)]
assert bbs_copy == bbs[len(to_delete):]
site_config["basebackup_count"] = 16
site_config["basebackup_age_days_max"] = 8
bbs_copy = list(bbs)
to_delete = self.pghoard.determine_backups_to_delete(
basebackups=bbs_copy, site_config=site_config)
# 3 of the backups are too old (start time + interval is over 8 days in the past)
assert len(bbs_copy) == 10
assert len(to_delete) == len(bbs) - len(bbs_copy)
assert to_delete == bbs[:len(to_delete)]
assert bbs_copy == bbs[len(to_delete):]
site_config["basebackup_count"] = 9
bbs_copy = list(bbs)
to_delete = self.pghoard.determine_backups_to_delete(
basebackups=bbs_copy, site_config=site_config)
# basebackup_count trumps backup age and backups are removed even though they're not too old
assert len(bbs_copy) == 9
assert len(to_delete) == len(bbs) - len(bbs_copy)
assert to_delete == bbs[:len(to_delete)]
assert bbs_copy == bbs[len(to_delete):]
site_config["basebackup_count"] = 16
site_config["basebackup_age_days_max"] = 2
site_config["basebackup_count_min"] = 6
bbs_copy = list(bbs)
to_delete = self.pghoard.determine_backups_to_delete(
basebackups=bbs_copy, site_config=site_config)
# basebackup_count_min ensures not that many backups are removed even though they're too old
assert len(bbs_copy) == 6
assert len(to_delete) == len(bbs) - len(bbs_copy)
assert to_delete == bbs[:len(to_delete)]
assert bbs_copy == bbs[len(to_delete):]
site_config["basebackup_count_min"] = 2
bbs_copy = list(bbs)
to_delete = self.pghoard.determine_backups_to_delete(
basebackups=bbs_copy, site_config=site_config)
# 3 of the backups are new enough (start time less than 3 days in the past)
assert len(bbs_copy) == 3
assert len(to_delete) == len(bbs) - len(bbs_copy)
assert to_delete == bbs[:len(to_delete)]
assert bbs_copy == bbs[len(to_delete):]
def test_local_refresh_backup_list_and_delete_old(self):
basebackup_storage_path = os.path.join(self.local_storage_dir,
"basebackup")
wal_storage_path = os.path.join(self.local_storage_dir, "xlog")
os.makedirs(basebackup_storage_path)
os.makedirs(wal_storage_path)
self.pghoard.set_state_defaults(self.test_site)
assert self.pghoard.get_remote_basebackups_info(self.test_site) == []
def write_backup_and_wal_files(what):
for bb, wals in what.items():
if bb:
bb_path = os.path.join(basebackup_storage_path, bb)
date_parts = [
int(part) for part in bb.replace("_", "-").split("-")
]
start_time = datetime.datetime(
*date_parts, tzinfo=datetime.timezone.utc)
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(bb_path + ".metadata", "w") as fp:
json.dump(
{
"start-wal-segment": wals[0],
"start-time": start_time.isoformat(),
}, fp)
for wal in wals:
with open(os.path.join(wal_storage_path, wal), "wb") as fp:
fp.write(b"something")
backups_and_wals = {
"2015-08-25_0": [
# NOTE: gap between this and next segment means that cleanup shouldn't find this
"000000010000000A000000FB",
],
"2015-08-25_1": [
"000000020000000A000000FD",
"000000020000000A000000FE",
],
"2015-08-25_2": [
"000000030000000A000000FF",
"000000030000000B00000000",
"000000030000000B00000001",
"000000040000000B00000002",
],
"2015-08-25_3": [
# Both of these should be saved
"000000040000000B00000003",
"000000040000000B00000004",
],
}
write_backup_and_wal_files(backups_and_wals)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 4
self.pghoard.refresh_backup_list_and_delete_old(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
assert len(os.listdir(wal_storage_path)) == 3
# Put all WAL segments between 1 and 9 in place to see that they're deleted and we don't try to go back
# any further from TLI 1. Note that timeline 3 is now "empty" so deletion shouldn't touch timelines 2
# or 1.
new_backups_and_wals = {
"": [
"000000020000000A000000FC",
"000000020000000A000000FD",
"000000020000000A000000FE",
"000000020000000A000000FF",
"000000020000000B00000000",
"000000020000000B00000001",
"000000020000000B00000002",
],
"2015-08-25_4": [
"000000040000000B00000005",
],
}
write_backup_and_wal_files(new_backups_and_wals)
assert len(os.listdir(wal_storage_path)) == 11
self.pghoard.refresh_backup_list_and_delete_old(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
expected_wal_count = len(backups_and_wals["2015-08-25_0"])
expected_wal_count += len(new_backups_and_wals[""])
expected_wal_count += len(new_backups_and_wals["2015-08-25_4"])
assert len(os.listdir(wal_storage_path)) == expected_wal_count
# Now put WAL files in place with no gaps anywhere
gapless_backups_and_wals = {
"2015-08-25_3": [
"000000030000000B00000003",
"000000040000000B00000004",
],
"2015-08-25_4": [
"000000040000000B00000005",
],
}
write_backup_and_wal_files(gapless_backups_and_wals)
assert len(os.listdir(wal_storage_path)) >= 10
self.pghoard.refresh_backup_list_and_delete_old(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
assert len(os.listdir(wal_storage_path)) == 1
def test_local_refresh_backup_list_and_delete_old_delta_format(self):
basebackup_storage_path = os.path.join(self.local_storage_dir,
"basebackup")
basebackup_delta_path = os.path.join(self.local_storage_dir,
"basebackup_delta")
os.makedirs(basebackup_storage_path)
os.makedirs(basebackup_delta_path)
self.pghoard.set_state_defaults(self.test_site)
assert self.pghoard.get_remote_basebackups_info(self.test_site) == []
def write_backup_files(what):
for bb, bb_data in what.items():
wal_start, hexdigests = bb_data
if bb:
bb_path = os.path.join(basebackup_storage_path, bb)
date_parts = [
int(part) for part in bb.replace("_", "-").split("-")
]
start_time = datetime.datetime(
*date_parts, tzinfo=datetime.timezone.utc)
metadata = {
"manifest": {
"snapshot_result": {
"state": {
"files": [{
"relative_path": h,
"hexdigest": h
} for h in hexdigests]
}
}
}
}
mtime = time.time()
blob = io.BytesIO(common.json_encode(metadata,
binary=True))
ti = tarfile.TarInfo(name=".pghoard_tar_metadata.json")
ti.size = len(blob.getbuffer())
ti.mtime = mtime
with open(bb_path, "wb") as fp:
with rohmufile.file_writer(
compression_algorithm="snappy",
compression_level=0,
fileobj=fp) as output_obj:
with tarfile.TarFile(fileobj=output_obj,
mode="w") as tar:
tar.addfile(ti, blob)
input_size = output_obj.tell()
for h in hexdigests:
with open(Path(basebackup_delta_path) / h, "w") as digest_file, \
open((Path(basebackup_delta_path) / (h + ".metadata")), "w") as digest_meta_file:
json.dump({}, digest_file)
json.dump({}, digest_meta_file)
with open(bb_path + ".metadata", "w") as fp:
json.dump(
{
"start-wal-segment": wal_start,
"start-time": start_time.isoformat(),
"format": BaseBackupFormat.delta_v1,
"compression-algorithm": "snappy",
"original-file-size": input_size
}, fp)
backups_and_delta = {
"2015-08-25_0": ("000000010000000A000000AA", [
"214967296374cae6f099e19910b33a0893f0abc62f50601baa2875ab055cd27b",
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
]),
"2015-08-25_1": [
"000000020000000A000000AB",
[
"214967296374cae6f099e19910b33a0893f0abc62f50601baa2875ab055cd27b"
]
],
"2015-08-25_2": [
"000000030000000A000000AC",
[
"214967296374cae6f099e19910b33a0893f0abc62f50601baa2875ab055cd27b"
]
],
"2015-08-25_3": [
"000000040000000B00000003",
[
"214967296374cae6f099e19910b33a0893f0abc62f50601baa2875ab055cd27b",
"4b65df4d0857bbbcb22aa086e02bd8414a9f3a484869f2b96ed7c62f3c4eb088"
]
],
}
write_backup_files(backups_and_delta)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 4
self.pghoard.refresh_backup_list_and_delete_old(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
left_delta_files = [
p for p in os.listdir(basebackup_delta_path)
if not p.endswith(".metadata")
]
assert sorted(left_delta_files) == [
"214967296374cae6f099e19910b33a0893f0abc62f50601baa2875ab055cd27b",
"4b65df4d0857bbbcb22aa086e02bd8414a9f3a484869f2b96ed7c62f3c4eb088"
]
new_delta_data = {
"2015-08-25_4": ("000000040000000B00000004", [
"fc61c91430dcb345001306ad513f103380c16896093a17868fc909aeda393559",
])
}
write_backup_files(new_delta_data)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 2
self.pghoard.refresh_backup_list_and_delete_old(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
left_delta_files = [
p for p in os.listdir(basebackup_delta_path)
if not p.endswith(".metadata")
]
assert sorted(left_delta_files) == [
"fc61c91430dcb345001306ad513f103380c16896093a17868fc909aeda393559",
]
def test_alert_files(self):
alert_file_path = os.path.join(self.config["alert_file_dir"],
"test_alert")
create_alert_file(self.pghoard.config, "test_alert")
assert os.path.exists(alert_file_path) is True
delete_alert_file(self.pghoard.config, "test_alert")
assert os.path.exists(alert_file_path) is False
def test_backup_state_file(self):
self.pghoard.write_backup_state_to_json_file()
state_path = self.config["json_state_file_path"]
with open(state_path, "r") as fp:
state = json.load(fp)
empty_state = {
"startup_time": self.pghoard.state["startup_time"],
"backup_sites": {},
"compressors": [{}] * self.config["compression"]["thread_count"],
"queues": {
"compression_queue": 0,
"transfer_queue": 0,
},
"served_files": {},
"transfer_agent_state": {},
"pg_receivexlogs": {},
"pg_basebackups": {},
"walreceivers": {},
}
assert empty_state == state
def test_startup_walk_for_missed_compressed_files(self):
compressed_wal_path, _ = self.pghoard.create_backup_site_paths(
self.test_site)
with open(
os.path.join(compressed_wal_path, "000000010000000000000004"),
"wb") as fp:
fp.write(b"foo")
with open(
os.path.join(compressed_wal_path,
"000000010000000000000004.metadata"), "wb") as fp:
fp.write(b"{}")
with open(os.path.join(compressed_wal_path, "0000000F.history"),
"wb") as fp:
fp.write(b"foo")
with open(
os.path.join(compressed_wal_path, "0000000F.history.metadata"),
"wb") as fp:
fp.write(b"{}")
with open(
os.path.join(compressed_wal_path,
"000000010000000000000004xyz"), "wb") as fp:
fp.write(b"foo")
with open(
os.path.join(compressed_wal_path,
"000000010000000000000004xyz.metadata"),
"wb") as fp:
fp.write(b"{}")
self.pghoard.startup_walk_for_missed_files()
assert self.pghoard.compression_queue.qsize() == 0
assert self.pghoard.transfer_queue.qsize() == 2
def test_startup_walk_for_missed_uncompressed_files(self):
compressed_wal_path, _ = self.pghoard.create_backup_site_paths(
self.test_site)
uncompressed_wal_path = compressed_wal_path + "_incoming"
with open(
os.path.join(uncompressed_wal_path,
"000000010000000000000004"), "wb") as fp:
fp.write(b"foo")
with open(os.path.join(uncompressed_wal_path, "00000002.history"),
"wb") as fp:
fp.write(b"foo")
with open(
os.path.join(uncompressed_wal_path,
"000000010000000000000004xyz"), "wb") as fp:
fp.write(b"foo")
self.pghoard.startup_walk_for_missed_files()
assert self.pghoard.compression_queue.qsize() == 2
assert self.pghoard.transfer_queue.qsize() == 0
class TestPGHoard(PGHoardTestCase):
def setup_method(self, method):
super().setup_method(method)
config_path = os.path.join(self.temp_dir, "pghoard.json")
self.config = create_json_conf(config_path, self.temp_dir, self.test_site)
backup_site_path = os.path.join(self.config["backup_location"], self.test_site)
self.compressed_xlog_path = os.path.join(backup_site_path, "xlog")
os.makedirs(self.compressed_xlog_path)
self.basebackup_path = os.path.join(backup_site_path, "basebackup")
os.makedirs(self.basebackup_path)
self.pghoard = PGHoard(config_path)
self.real_check_pg_server_version = self.pghoard.check_pg_server_version
self.pghoard.check_pg_server_version = Mock(return_value="psql (PostgreSQL) 9.4.4")
self.real_check_pg_versions_ok = self.pghoard.check_pg_versions_ok
self.pghoard.check_pg_versions_ok = Mock(return_value=True)
def teardown_method(self, method):
self.pghoard.quit()
self.pghoard.check_pg_server_version = self.real_check_pg_server_version
self.pghoard.check_pg_versions_ok = self.real_check_pg_versions_ok
super().teardown_method(method)
@patch("subprocess.check_output")
def test_handle_site(self, subprocess_mock):
subprocess_mock.return_value = b"""\
systemid|6222667313856416063
timeline|1
xlogpos|0/B003760
dbname|"""
self.pghoard.handle_site(self.test_site, self.config["backup_sites"][self.test_site])
assert self.pghoard.receivexlogs == {}
assert len(self.pghoard.time_of_last_backup_check) == 1
def test_get_local_basebackups_info(self):
assert self.pghoard.get_remote_basebackups_info(self.test_site) == []
bb_path = os.path.join(self.basebackup_path, "2015-07-03_0")
# Handle case where metadata file does not exist
assert self.pghoard.get_remote_basebackups_info(self.test_site) == []
metadata_file_path = bb_path + ".metadata"
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(metadata_file_path, "w") as fp:
json.dump({"start-time": "2015-07-03 12:00:00"}, fp)
available_backup = self.pghoard.get_remote_basebackups_info(self.test_site)[0]
assert available_backup["name"] == "2015-07-03_0"
assert available_backup["metadata"] == {"start-time": "2015-07-03 12:00:00"}
bb_path = os.path.join(self.basebackup_path, "2015-07-02_9")
metadata_file_path = bb_path + ".metadata"
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(metadata_file_path, "w") as fp:
json.dump({"start-time": "2015-07-02 12:00:00"}, fp)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert basebackups[0]["name"] == "2015-07-02_9"
assert basebackups[1]["name"] == "2015-07-03_0"
bb_path = os.path.join(self.basebackup_path, "2015-07-02_10")
metadata_file_path = bb_path + ".metadata"
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(metadata_file_path, "w") as fp:
json.dump({"start-time": "2015-07-02 22:00:00"}, fp)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert basebackups[0]["name"] == "2015-07-02_9"
assert basebackups[1]["name"] == "2015-07-02_10"
assert basebackups[2]["name"] == "2015-07-03_0"
def test_local_check_backup_count_and_state(self):
self.pghoard.set_state_defaults(self.test_site)
assert self.pghoard.get_remote_basebackups_info(self.test_site) == []
def write_backup_and_wal_files(what):
for bb, wals in what.items():
if bb:
bb_path = os.path.join(self.basebackup_path, bb)
with open(bb_path, "wb") as fp:
fp.write(b"something")
with open(bb_path + ".metadata", "w") as fp:
json.dump({
"start-wal-segment": wals[0],
"start-time": bb,
}, fp)
for wal in wals:
with open(os.path.join(self.compressed_xlog_path, wal), "wb") as fp:
fp.write(b"something")
backups_and_wals = {
"2015-08-25_0": [
# NOTE: gap between this and next segment means that cleanup shouldn't find this
"000000010000000A000000FB",
],
"2015-08-25_1": [
"000000020000000A000000FD",
"000000020000000A000000FE",
],
"2015-08-25_2": [
"000000030000000A000000FF",
"000000030000000B00000000",
"000000030000000B00000001",
"000000040000000B00000002",
],
"2015-08-25_3": [
# Both of these should be saved
"000000040000000B00000003",
"000000040000000B00000004",
],
}
write_backup_and_wal_files(backups_and_wals)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 4
self.pghoard.check_backup_count_and_state(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
assert len(os.listdir(self.compressed_xlog_path)) == 3
# Put all WAL segments between 1 and 9 in place to see that they're deleted and we don't try to go back
# any further from TLI 1. Note that timeline 3 is now "empty" so deletion shouldn't touch timelines 2
# or 1.
new_backups_and_wals = {
"": [
"000000020000000A000000FC",
"000000020000000A000000FD",
"000000020000000A000000FE",
"000000020000000A000000FF",
"000000020000000B00000000",
"000000020000000B00000001",
"000000020000000B00000002",
],
"2015-08-25_4": [
"000000040000000B00000005",
],
}
write_backup_and_wal_files(new_backups_and_wals)
assert len(os.listdir(self.compressed_xlog_path)) == 11
self.pghoard.check_backup_count_and_state(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
expected_wal_count = len(backups_and_wals["2015-08-25_0"])
expected_wal_count += len(new_backups_and_wals[""])
expected_wal_count += len(new_backups_and_wals["2015-08-25_4"])
assert len(os.listdir(self.compressed_xlog_path)) == expected_wal_count
# Now put WAL files in place with no gaps anywhere
gapless_backups_and_wals = {
"2015-08-25_3": [
"000000030000000B00000003",
"000000040000000B00000004",
],
"2015-08-25_4": [
"000000040000000B00000005",
],
}
write_backup_and_wal_files(gapless_backups_and_wals)
assert len(os.listdir(self.compressed_xlog_path)) >= 10
self.pghoard.check_backup_count_and_state(self.test_site)
basebackups = self.pghoard.get_remote_basebackups_info(self.test_site)
assert len(basebackups) == 1
assert len(os.listdir(self.compressed_xlog_path)) == 1
def test_alert_files(self):
alert_file_path = os.path.join(self.temp_dir, "test_alert")
create_alert_file(self.pghoard.config, "test_alert")
assert os.path.exists(alert_file_path) is True
delete_alert_file(self.pghoard.config, "test_alert")
assert os.path.exists(alert_file_path) is False
def test_backup_state_file(self):
self.pghoard.write_backup_state_to_json_file()
state_path = os.path.join(self.temp_dir, "state.json")
thread_count = 5
with open(state_path, "r") as fp:
state = json.load(fp)
empty_state = {
"startup_time": self.pghoard.state["startup_time"],
"backup_sites": {},
"compressors": [{}] * thread_count,
"queues": {
"compression_queue": 0,
"transfer_queue": 0,
},
"transfer_agents": [{}] * thread_count,
"pg_receivexlogs": {},
"pg_basebackups": {},
}
assert empty_state == state
def test_startup_walk_for_missed_files(self):
with open(os.path.join(self.compressed_xlog_path, "000000010000000000000004"), "wb") as fp:
fp.write(b"foo")
self.pghoard.startup_walk_for_missed_files()
assert self.pghoard.compression_queue.qsize() == 1
def pghoard_base(db, tmpdir, request, compression="snappy", # pylint: disable=redefined-outer-name
transfer_count=None, metrics_cfg=None):
test_site = request.function.__name__
if os.environ.get("pghoard_test_walreceiver"):
active_backup_mode = "walreceiver"
else:
active_backup_mode = "pg_receivexlog"
if compression == "snappy" and not snappy:
compression = "lzma"
config = {
"alert_file_dir": os.path.join(str(tmpdir), "alerts"),
"backup_location": os.path.join(str(tmpdir), "backupspool"),
"backup_sites": {
test_site: {
"active_backup_mode": active_backup_mode,
"basebackup_count": 2,
"basebackup_interval_hours": 24,
"pg_bin_directory": db.pgbin,
"pg_data_directory": db.pgdata,
"nodes": [db.user],
"object_storage": {
"storage_type": "local",
"directory": os.path.join(str(tmpdir), "backups"),
},
},
},
"compression": {
"algorithm": compression,
},
"http_address": "127.0.0.1",
"http_port": random.randint(1024, 32000),
"json_state_file_path": tmpdir.join("pghoard_state.json").strpath,
"maintenance_mode_file": tmpdir.join("maintenance_mode_file").strpath,
# Set process count to 1 to avoid launching subprocesses during basebackup tests.
# The new processes would be created with fork, which doesn't work properly due to
# all the fds and other things that are created during typical test setup. There
# is separate test case that executes the multiprocess version.
"restore_process_count": 1,
"tar_executable": "tar",
}
if metrics_cfg is not None:
config.update(metrics_cfg)
if transfer_count is not None:
config["transfer"] = {"thread_count": transfer_count}
confpath = os.path.join(str(tmpdir), "config.json")
with open(confpath, "w") as fp:
json.dump(config, fp)
backup_site_path = os.path.join(config["backup_location"], test_site)
basebackup_path = os.path.join(backup_site_path, "basebackup")
backup_xlog_path = os.path.join(backup_site_path, "xlog")
backup_timeline_path = os.path.join(backup_site_path, "timeline")
os.makedirs(config["alert_file_dir"])
os.makedirs(basebackup_path)
os.makedirs(backup_xlog_path)
os.makedirs(backup_timeline_path)
pgh = PGHoard(confpath)
pgh.test_site = test_site
pgh.start_threads_on_startup()
if compression == "snappy":
pgh.Compressor = snappy.StreamCompressor
else:
pgh.Compressor = lambda: lzma.LZMACompressor(preset=0) # pylint: disable=redefined-variable-type
time.sleep(0.05) # Hack to give the server time to start up
yield pgh
pgh.quit()
