def test_list_empty_basebackups(self, pghoard, http_restore, capsys): # pylint: disable=redefined-outer-name
# List with direct HttpRestore access
assert http_restore.list_basebackups() == []
http_restore.show_basebackup_list()
out, _ = capsys.readouterr()
assert pghoard.test_site in out
# list using restore command over http
Restore().run([
"list-basebackups-http",
"--host",
"localhost",
"--port",
str(pghoard.config["http_port"]),
"--site",
pghoard.test_site,
])
out, _ = capsys.readouterr()
assert pghoard.test_site in out
# list using restore command with direct object storage access
Restore().run([
"list-basebackups",
"--config",
pghoard.config_path,
"--site",
pghoard.test_site,
])
out, _ = capsys.readouterr()
assert pghoard.test_site in out
def _test_restore_basebackup(self, db, pghoard, tmpdir):
backup_out = tmpdir.join("test-restore").strpath
# Restoring to empty directory works
os.makedirs(backup_out)
Restore().run([
"get-basebackup",
"--config", pghoard.config_path,
"--site", pghoard.test_site,
"--target-dir", backup_out,
])
# Restoring on top of another $PGDATA doesn't
with pytest.raises(RestoreError) as excinfo:
Restore().run([
"get-basebackup",
"--config", pghoard.config_path,
"--site", pghoard.test_site,
"--target-dir", backup_out,
])
assert "--overwrite not specified" in str(excinfo.value)
# Until we use the --overwrite flag
Restore().run([
"get-basebackup",
"--config", pghoard.config_path,
"--site", pghoard.test_site,
"--target-dir", backup_out,
"--overwrite",
])
check_call([os.path.join(db.pgbin, "pg_controldata"), backup_out])
def test_basebackups(self, capsys, db, pghoard, tmpdir):
pghoard.create_backup_site_paths(pghoard.test_site)
r_conn = deepcopy(db.user)
r_conn["dbname"] = "replication"
r_conn["replication"] = True
conn_str = create_connection_string(r_conn)
basebackup_path = os.path.join(pghoard.config["backup_location"],
pghoard.test_site, "basebackup")
q = Queue()
pghoard.create_basebackup(pghoard.test_site, conn_str, basebackup_path,
q)
result = q.get(timeout=60)
assert result["success"]
pghoard.config["backup_sites"][
pghoard.test_site]["stream_compression"] = True
pghoard.create_basebackup(pghoard.test_site, conn_str, basebackup_path,
q)
result = q.get(timeout=60)
assert result["success"]
if not pghoard.config["backup_sites"][
pghoard.test_site]["object_storage"]:
assert os.path.exists(
pghoard.basebackups[pghoard.test_site].target_basebackup_path)
# make sure it shows on the list
Restore().run([
"list-basebackups",
"--config",
pghoard.config_path,
"--site",
pghoard.test_site,
])
out, _ = capsys.readouterr()
assert pghoard.test_site in out
# try downloading it
backup_out = str(tmpdir.join("test-restore"))
os.makedirs(backup_out)
with pytest.raises(RestoreError) as excinfo:
Restore().run([
"get-basebackup",
"--config",
pghoard.config_path,
"--site",
pghoard.test_site,
"--target-dir",
backup_out,
])
assert "--overwrite not specified" in str(excinfo.value)
Restore().run([
"get-basebackup",
"--config",
pghoard.config_path,
"--site",
pghoard.test_site,
"--target-dir",
backup_out,
"--overwrite",
])
def _test_create_basebackup(self, capsys, db, pghoard, mode):
pghoard.create_backup_site_paths(pghoard.test_site)
basebackup_path = os.path.join(pghoard.config["backup_location"],
pghoard.test_site, "basebackup")
q = Queue()
pghoard.config["backup_sites"][
pghoard.test_site]["basebackup_mode"] = mode
pghoard.create_basebackup(pghoard.test_site, db.user, basebackup_path,
q)
result = q.get(timeout=60)
assert result["success"]
# make sure it shows on the list
Restore().run([
"list-basebackups",
"--config",
pghoard.config_path,
"--site",
pghoard.test_site,
"--verbose",
])
out, _ = capsys.readouterr()
assert pghoard.test_site in out
assert "pg-version" in out
def _test_create_basebackup(self,
capsys,
db,
pghoard,
mode,
replica=False,
active_backup_mode='archive_command'):
pghoard.create_backup_site_paths(pghoard.test_site)
basebackup_path = os.path.join(pghoard.config["backup_location"],
pghoard.test_site, "basebackup")
q = Queue()
pghoard.config["backup_sites"][
pghoard.test_site]["basebackup_mode"] = mode
pghoard.config["backup_sites"][
pghoard.test_site]["active_backup_mode"] = active_backup_mode
pghoard.create_basebackup(pghoard.test_site, db.user, basebackup_path,
q)
result = q.get(timeout=60)
assert result["success"]
# make sure it shows on the list
Restore().run([
"list-basebackups",
"--config",
pghoard.config_path,
"--site",
pghoard.test_site,
"--verbose",
])
out, _ = capsys.readouterr()
assert pghoard.test_site in out
assert "pg-version" in out
assert "start-wal-segment" in out
if mode == "local-tar":
assert "end-time" in out
if replica is False:
assert "end-wal-segment" in out
storage_config = common.get_object_storage_config(
pghoard.config, pghoard.test_site)
storage = get_transfer(storage_config)
backups = storage.list_path(
os.path.join(
pghoard.config["backup_sites"][pghoard.test_site]["prefix"],
"basebackup"))
for backup in backups:
assert "start-wal-segment" in backup["metadata"]
assert "start-time" in backup["metadata"]
assert dateutil.parser.parse(
backup["metadata"]["start-time"]).tzinfo # pylint: disable=no-member
if mode == "local-tar":
if replica is False:
assert "end-wal-segment" in backup["metadata"]
assert "end-time" in backup["metadata"]
assert dateutil.parser.parse(
backup["metadata"]["end-time"]).tzinfo # pylint: disable=no-member
def test_recovery_targets(self, tmpdir):
r = Restore()
r._load_config = Mock() # pylint: disable=protected-access
r._get_object_storage = Mock() # pylint: disable=protected-access
with pytest.raises(RestoreError) as excinfo:
r.run(args=[
"get-basebackup",
"--config=" + str(tmpdir),
"--target-dir=" + str(tmpdir),
"--site=test",
"--recovery-target-action=promote",
"--recovery-target-name=foobar",
"--recovery-target-xid=42",
])
assert "at most one" in str(excinfo.value)
with pytest.raises(RestoreError) as excinfo:
r.run(args=[
"get-basebackup",
"--config=" + str(tmpdir),
"--target-dir=" + str(tmpdir),
"--site=test",
"--recovery-target-action=promote",
"--recovery-target-time=foobar",
])
assert "recovery_target_time 'foobar'" in str(excinfo.value)
def test_find_nearest_backup(self):
r = Restore()
r.storage = Mock()
basebackups = [
{
"name": "2015-02-12_0",
"size": 42,
"metadata": {
"start-time": "2015-02-12T14:07:19+00:00"
},
},
{
"name": "2015-02-13_0",
"size": 42 * 1024 * 1024,
"metadata": {
"start-time": "2015-02-13T14:07:19+00:00"
},
},
]
r.storage.list_basebackups = Mock(return_value=basebackups)
assert r._find_nearest_basebackup()["name"] == "2015-02-13_0" # pylint: disable=protected-access
recovery_time = datetime.datetime(2015, 2, 1)
recovery_time = recovery_time.replace(tzinfo=datetime.timezone.utc)
with pytest.raises(RestoreError):
r._find_nearest_basebackup(recovery_time) # pylint: disable=protected-access
recovery_time = datetime.datetime(2015, 2, 12, 14, 20)
recovery_time = recovery_time.replace(tzinfo=datetime.timezone.utc)
assert r._find_nearest_basebackup(
recovery_time)["name"] == "2015-02-12_0" # pylint: disable=protected-access
def test_recovery_targets(self, tmpdir):
config_file = tmpdir.join("conf.json").strpath
write_json_file(config_file, {"backup_sites": {"test": {}}})
r = Restore()
r._get_object_storage = Mock() # pylint: disable=protected-access
with pytest.raises(RestoreError) as excinfo:
r.run(args=[
"get-basebackup",
"--config", config_file,
"--target-dir", tmpdir.strpath,
"--site=test",
"--recovery-target-action=promote",
"--recovery-target-name=foobar",
"--recovery-target-xid=42",
])
assert "at most one" in str(excinfo.value)
with pytest.raises(RestoreError) as excinfo:
r.run(args=[
"get-basebackup",
"--config", config_file,
"--target-dir", tmpdir.strpath,
"--site=test",
"--recovery-target-action=promote",
"--recovery-target-time=foobar",
])
assert "recovery_target_time 'foobar'" in str(excinfo.value)
def test_recovery_targets(self, tmpdir):
config_file = tmpdir.join("conf.json").strpath
# Instantiate a fake PG data directory
pg_data_directory = os.path.join(str(self.temp_dir), "PG_DATA_DIRECTORY")
os.makedirs(pg_data_directory)
open(os.path.join(pg_data_directory, "PG_VERSION"), "w").write("9.6")
write_json_file(config_file, {"backup_sites": {"test": {"pg_data_directory": pg_data_directory}}})
r = Restore()
r._get_object_storage = Mock() # pylint: disable=protected-access
with pytest.raises(RestoreError) as excinfo:
r.run(args=[
"get-basebackup",
"--config", config_file,
"--target-dir", tmpdir.strpath,
"--site=test",
"--recovery-target-action=promote",
"--recovery-target-name=foobar",
"--recovery-target-xid=42",
])
assert "at most one" in str(excinfo.value)
with pytest.raises(RestoreError) as excinfo:
r.run(args=[
"get-basebackup",
"--config", config_file,
"--target-dir", tmpdir.strpath,
"--site=test",
"--recovery-target-action=promote",
"--recovery-target-time=foobar",
])
assert "recovery_target_time 'foobar'" in str(excinfo.value)
def test_find_nearest_backup(self):
r = Restore()
r.storage = Mock()
basebackups = [
{
"name": "2015-02-12_0",
"size": 42,
"metadata": {"start-time": "2015-02-12T14:07:19+00:00"},
},
{
"name": "2015-02-13_0",
"size": 42 * 1024 * 1024,
"metadata": {"start-time": "2015-02-13T14:07:19+00:00"},
},
]
r.storage.list_basebackups = Mock(return_value=basebackups)
assert r._find_nearest_basebackup()["name"] == "2015-02-13_0" # pylint: disable=protected-access
recovery_time = datetime.datetime(2015, 2, 1)
recovery_time = recovery_time.replace(tzinfo=datetime.timezone.utc)
with pytest.raises(RestoreError):
r._find_nearest_basebackup(recovery_time) # pylint: disable=protected-access
recovery_time = datetime.datetime(2015, 2, 12, 14, 20)
recovery_time = recovery_time.replace(tzinfo=datetime.timezone.utc)
assert r._find_nearest_basebackup(recovery_time)["name"] == "2015-02-12_0" # pylint: disable=protected-access
def test_recovery_targets(self, tmpdir):
config_file = tmpdir.join("conf.json").strpath
# Instantiate a fake PG data directory
pg_data_directory = os.path.join(str(self.temp_dir), "PG_DATA_DIRECTORY")
os.makedirs(pg_data_directory)
open(os.path.join(pg_data_directory, "PG_VERSION"), "w").write("9.6")
write_json_file(config_file, {"backup_sites": {"test": {"pg_data_directory": pg_data_directory}}})
r = Restore()
r._get_object_storage = Mock() # pylint: disable=protected-access
with pytest.raises(RestoreError) as excinfo:
r.run(args=[
"get-basebackup",
"--config", config_file,
"--target-dir", tmpdir.strpath,
"--site=test",
"--recovery-target-action=promote",
"--recovery-target-name=foobar",
"--recovery-target-xid=42",
])
assert "at most one" in str(excinfo.value)
with pytest.raises(RestoreError) as excinfo:
r.run(args=[
"get-basebackup",
"--config", config_file,
"--target-dir", tmpdir.strpath,
"--site=test",
"--recovery-target-action=promote",
"--recovery-target-time=foobar",
])
assert "recovery_target_time 'foobar'" in str(excinfo.value)
def _test_create_basebackup(self, capsys, db, pghoard, mode, replica=False, active_backup_mode="archive_command"):
pghoard.create_backup_site_paths(pghoard.test_site)
basebackup_path = os.path.join(pghoard.config["backup_location"], pghoard.test_site, "basebackup")
q = Queue()
pghoard.config["backup_sites"][pghoard.test_site]["basebackup_mode"] = mode
pghoard.config["backup_sites"][pghoard.test_site]["active_backup_mode"] = active_backup_mode
now = datetime.datetime.now(datetime.timezone.utc)
metadata = {
"backup-reason": "scheduled",
"backup-decision-time": now.isoformat(),
"normalized-backup-time": now.isoformat(),
}
pghoard.create_basebackup(pghoard.test_site, db.user, basebackup_path, q, metadata)
result = q.get(timeout=60)
assert result["success"]
# make sure it shows on the list
Restore().run([
"list-basebackups",
"--config",
pghoard.config_path,
"--site",
pghoard.test_site,
"--verbose",
])
out, _ = capsys.readouterr()
assert pghoard.test_site in out
assert "pg-version" in out
assert "start-wal-segment" in out
if mode in {BaseBackupMode.local_tar, BaseBackupMode.delta}:
assert "end-time" in out
if replica is False:
assert "end-wal-segment" in out
storage_config = common.get_object_storage_config(pghoard.config, pghoard.test_site)
storage = get_transfer(storage_config)
backups = storage.list_path(os.path.join(pghoard.config["backup_sites"][pghoard.test_site]["prefix"], "basebackup"))
for backup in backups:
assert "start-wal-segment" in backup["metadata"]
assert "start-time" in backup["metadata"]
assert dateutil.parser.parse(backup["metadata"]["start-time"]).tzinfo # pylint: disable=no-member
assert backup["metadata"]["backup-reason"] == "scheduled"
assert backup["metadata"]["backup-decision-time"] == now.isoformat()
assert backup["metadata"]["normalized-backup-time"] == now.isoformat()
if mode in {BaseBackupMode.local_tar, BaseBackupMode.delta}:
if replica is False:
assert "end-wal-segment" in backup["metadata"]
assert "end-time" in backup["metadata"]
assert dateutil.parser.parse(backup["metadata"]["end-time"]).tzinfo # pylint: disable=no-member
def test_basebackups_tablespaces(self, capsys, db, pghoard, tmpdir):
# Create a test tablespace for this instance, but make sure we drop it at the end of the test as the
# database we use is shared by all test cases, and tablespaces are a global concept so the test
# tablespace could interfere with other tests
tspath = tmpdir.join("extra-ts").strpath
os.makedirs(tspath)
conn_str = pgutil.create_connection_string(db.user)
conn = psycopg2.connect(conn_str)
conn.autocommit = True
cursor = conn.cursor()
cursor.execute("CREATE TABLESPACE tstest LOCATION %s", [tspath])
r_db, r_conn = None, None
try:
cursor.execute(
"CREATE TABLE tstest (id BIGSERIAL PRIMARY KEY, value BIGINT) TABLESPACE tstest"
)
cursor.execute(
"INSERT INTO tstest (value) SELECT * FROM generate_series(1, 1000)"
)
cursor.execute("CHECKPOINT")
cursor.execute(
"SELECT oid, pg_tablespace_location(oid) FROM pg_tablespace WHERE spcname = 'tstest'"
)
res = cursor.fetchone()
assert res[1] == tspath
# Start receivexlog since we want the WALs to be able to restore later on
wal_directory = os.path.join(pghoard.config["backup_location"],
pghoard.test_site, "xlog_incoming")
makedirs(wal_directory, exist_ok=True)
pghoard.receivexlog_listener(pghoard.test_site, db.user,
wal_directory)
if conn.server_version >= 100000:
cursor.execute("SELECT txid_current(), pg_switch_wal()")
else:
cursor.execute("SELECT txid_current(), pg_switch_xlog()")
self._test_create_basebackup(capsys, db, pghoard, "local-tar")
if conn.server_version >= 100000:
cursor.execute("SELECT txid_current(), pg_switch_wal()")
cursor.execute("SELECT txid_current(), pg_switch_wal()")
else:
cursor.execute("SELECT txid_current(), pg_switch_xlog()")
cursor.execute("SELECT txid_current(), pg_switch_xlog()")
backup_out = tmpdir.join("test-restore").strpath
backup_ts_out = tmpdir.join("test-restore-tstest").strpath
# Tablespaces are extracted to their previous absolute paths by default, but the path must be empty
# and it isn't as it's still used by the running PG
with pytest.raises(RestoreError) as excinfo:
Restore().run([
"get-basebackup",
"--config",
pghoard.config_path,
"--site",
pghoard.test_site,
"--target-dir",
backup_out,
])
assert "Tablespace 'tstest' target directory" in str(excinfo.value)
assert "not empty" in str(excinfo.value)
# We can't restore tablespaces to non-existent directories either
with pytest.raises(RestoreError) as excinfo:
Restore().run([
"get-basebackup",
"--config",
pghoard.config_path,
"--site",
pghoard.test_site,
"--target-dir",
backup_out,
"--tablespace-dir",
"tstest={}".format(backup_ts_out),
])
assert "Tablespace 'tstest' target directory" in str(excinfo.value)
assert "does not exist" in str(excinfo.value)
os.makedirs(backup_ts_out)
# We can't restore if the directory isn't writable
os.chmod(backup_ts_out, 0o500)
with pytest.raises(RestoreError) as excinfo:
Restore().run([
"get-basebackup",
"--config",
pghoard.config_path,
"--site",
pghoard.test_site,
"--target-dir",
backup_out,
"--tablespace-dir",
"tstest={}".format(backup_ts_out),
])
assert "Tablespace 'tstest' target directory" in str(excinfo.value)
assert "empty, but not writable" in str(excinfo.value)
os.chmod(backup_ts_out, 0o700)
# We can't proceed if we request mappings for non-existent tablespaces
backup_other_out = tmpdir.join("test-restore-other").strpath
os.makedirs(backup_other_out)
with pytest.raises(RestoreError) as excinfo:
Restore().run([
"get-basebackup",
"--config",
pghoard.config_path,
"--site",
pghoard.test_site,
"--target-dir",
backup_out,
"--tablespace-dir",
"tstest={}".format(backup_ts_out),
"--tablespace-dir",
"other={}".format(backup_other_out),
])
assert "Tablespace mapping for ['other'] was requested, but" in str(
excinfo.value)
# Now, finally, everything should be valid and we can proceed with restore
Restore().run([
"get-basebackup",
"--config",
pghoard.config_path,
"--site",
pghoard.test_site,
"--restore-to-master",
"--target-dir",
backup_out,
"--tablespace-dir",
"tstest={}".format(backup_ts_out),
])
# Adjust the generated recovery.conf to point pghoard_postgres_command to our instance
new_py_restore_cmd = "PYTHONPATH={} python3 -m pghoard.postgres_command --mode restore".format(
os.path.dirname(os.path.dirname(__file__)))
new_go_restore_cmd = "{}/pghoard_postgres_command_go --mode restore".format(
os.path.dirname(os.path.dirname(__file__)))
with open(os.path.join(backup_out, "recovery.conf"), "r+") as fp:
rconf = fp.read()
rconf = rconf.replace(
"pghoard_postgres_command_go --mode restore",
new_go_restore_cmd)
rconf = rconf.replace(
"pghoard_postgres_command --mode restore",
new_py_restore_cmd)
fp.seek(0)
fp.write(rconf)
r_db = PGTester(backup_out)
r_db.user = dict(db.user, host=backup_out)
r_db.run_pg()
r_conn_str = pgutil.create_connection_string(r_db.user)
# Wait for PG to start up
start_time = time.monotonic()
while True:
try:
r_conn = psycopg2.connect(r_conn_str)
break
except psycopg2.OperationalError as ex:
if "starting up" in str(ex):
assert time.monotonic() - start_time <= 10
time.sleep(1)
else:
raise
r_cursor = r_conn.cursor()
# Make sure the tablespace is defined and points to the right (new) path
r_cursor.execute(
"SELECT oid, pg_tablespace_location(oid) FROM pg_tablespace WHERE spcname = 'tstest'"
)
r_res = r_cursor.fetchone()
assert r_res[1] == backup_ts_out
# We should be able to read from the table in the tablespace and the values should match what we stored before
r_cursor.execute("SELECT id FROM tstest")
r_res = r_cursor.fetchall()
cursor.execute("SELECT id FROM tstest")
orig_res = cursor.fetchall()
assert r_res == orig_res
finally:
if r_conn:
r_conn.close()
if r_db:
r_db.kill(force=True)
cursor.execute("DROP TABLE IF EXISTS tstest")
cursor.execute("DROP TABLESPACE tstest")
conn.close()
def _test_restore_basebackup(self,
db,
pghoard,
tmpdir,
active_backup_mode="archive_command"):
backup_out = tmpdir.join("test-restore").strpath
# Restoring to empty directory works
os.makedirs(backup_out)
Restore().run([
"get-basebackup",
"--config",
pghoard.config_path,
"--site",
pghoard.test_site,
"--target-dir",
backup_out,
])
# Restoring on top of another $PGDATA doesn't
with pytest.raises(RestoreError) as excinfo:
Restore().run([
"get-basebackup",
"--config",
pghoard.config_path,
"--site",
pghoard.test_site,
"--target-dir",
backup_out,
])
assert "--overwrite not specified" in str(excinfo.value)
# Until we use the --overwrite flag
Restore().run([
"get-basebackup",
"--config",
pghoard.config_path,
"--site",
pghoard.test_site,
"--target-dir",
backup_out,
"--overwrite",
])
check_call([os.path.join(db.pgbin, "pg_controldata"), backup_out])
# TODO: check that the backup is valid
# there should only be a single backup so lets compare what was in the metadata with what
# was in the backup label
storage_config = common.get_object_storage_config(
pghoard.config, pghoard.test_site)
storage = get_transfer(storage_config)
backups = storage.list_path(
os.path.join(
pghoard.config["backup_sites"][pghoard.test_site]["prefix"],
"basebackup"))
# lets grab the backup label details for what we restored
pgb = PGBaseBackup(config=None,
site="foosite",
connection_info=None,
basebackup_path=None,
compression_queue=None,
transfer_queue=None,
metrics=metrics.Metrics(statsd={}))
path = os.path.join(backup_out, "backup_label")
with open(path, "r") as myfile:
data = myfile.read()
start_wal_segment, start_time = pgb.parse_backup_label(data)
assert start_wal_segment == backups[0]['metadata']['start-wal-segment']
assert start_time == backups[0]['metadata']['start-time']
# for a standalone hot backup, the start wal file will be in the pg_xlog / pg_wal directory
wal_dir = "pg_xlog"
if float(db.pgver) >= float("10.0"):
wal_dir = "pg_wal"
path = os.path.join(backup_out, wal_dir,
backups[0]['metadata']['start-wal-segment'])
if active_backup_mode == "standalone_hot_backup":
assert os.path.isfile(path) is True
else:
assert os.path.isfile(path) is False