def test_get_encrypted_archived_file(self, pghoard):
xlog_seg = "000000090000000000000010"
content = wal_header_for_file(xlog_seg)
compressor = pghoard.Compressor()
compressed_content = compressor.compress(content) + (compressor.flush() or b"")
encryptor = Encryptor(CONSTANT_TEST_RSA_PUBLIC_KEY)
encrypted_content = encryptor.update(compressed_content) + encryptor.finalize()
pgdata = os.path.dirname(pghoard.config["backup_sites"][pghoard.test_site]["pg_xlog_directory"])
archive_path = os.path.join(pghoard.test_site, "xlog", xlog_seg)
metadata = {
"compression-algorithm": pghoard.config["compression"]["algorithm"],
"original-file-size": len(content),
"encryption-key-id": "testkey",
}
store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
store.store_file_from_memory(archive_path, encrypted_content, metadata=metadata)
pghoard.webserver.config["backup_sites"][pghoard.test_site]["encryption_keys"] = {
"testkey": {
"public": CONSTANT_TEST_RSA_PUBLIC_KEY,
"private": CONSTANT_TEST_RSA_PRIVATE_KEY,
},
}
restore_target = os.path.join(pgdata, "pg_xlog", xlog_seg)
restore_command(site=pghoard.test_site, xlog=xlog_seg, output=restore_target,
host="127.0.0.1", port=pghoard.config["http_port"])
assert os.path.exists(restore_target)
with open(restore_target, "rb") as fp:
restored_data = fp.read()
assert content == restored_data
def test_get_encrypted_archived_file(self, pghoard):
wal_seg = "000000090000000000000010"
content = wal_header_for_file(wal_seg)
compressor = pghoard.Compressor()
compressed_content = compressor.compress(content) + (compressor.flush() or b"")
encryptor = Encryptor(CONSTANT_TEST_RSA_PUBLIC_KEY)
encrypted_content = encryptor.update(compressed_content) + encryptor.finalize()
wal_dir = get_pg_wal_directory(pghoard.config["backup_sites"][pghoard.test_site])
archive_path = os.path.join(pghoard.test_site, "xlog", wal_seg)
metadata = {
"compression-algorithm": pghoard.config["compression"]["algorithm"],
"original-file-size": len(content),
"encryption-key-id": "testkey",
}
store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
store.store_file_from_memory(archive_path, encrypted_content, metadata=metadata)
pghoard.webserver.config["backup_sites"][pghoard.test_site]["encryption_keys"] = {
"testkey": {
"public": CONSTANT_TEST_RSA_PUBLIC_KEY,
"private": CONSTANT_TEST_RSA_PRIVATE_KEY,
},
}
restore_target = os.path.join(wal_dir, wal_seg)
restore_command(site=pghoard.test_site, xlog=wal_seg, output=restore_target,
host="127.0.0.1", port=pghoard.config["http_port"])
assert os.path.exists(restore_target)
with open(restore_target, "rb") as fp:
restored_data = fp.read()
assert content == restored_data
def test_get_archived_file(self, pghoard):
wal_seg_prev_tli = "00000001000000000000000F"
wal_seg = "00000002000000000000000F"
wal_file = "xlog/{}".format(wal_seg)
# NOTE: create WAL header for the "previous" timeline, this should be accepted
content = wal_header_for_file(wal_seg_prev_tli)
wal_dir = get_pg_wal_directory(
pghoard.config["backup_sites"][pghoard.test_site])
archive_path = os.path.join(pghoard.test_site, wal_file)
compressor = pghoard.Compressor()
compressed_content = compressor.compress(content) + (
compressor.flush() or b"")
metadata = {
"compression-algorithm":
pghoard.config["compression"]["algorithm"],
"original-file-size": len(content),
}
store = pghoard.transfer_agents[0].get_object_storage(
pghoard.test_site)
store.store_file_from_memory(archive_path,
compressed_content,
metadata=metadata)
restore_command(site=pghoard.test_site,
xlog=wal_seg,
output=None,
host="127.0.0.1",
port=pghoard.config["http_port"])
restore_target = os.path.join(wal_dir, wal_seg)
restore_command(site=pghoard.test_site,
xlog=wal_seg,
output=restore_target,
host="127.0.0.1",
port=pghoard.config["http_port"])
assert os.path.exists(restore_target) is True
with open(restore_target, "rb") as fp:
restored_data = fp.read()
assert content == restored_data
# test the same thing using restore as 'pghoard_postgres_command'
tmp_out = os.path.join(wal_dir, restore_target + ".cmd")
postgres_command.main([
"--host",
"localhost",
"--port",
str(pghoard.config["http_port"]),
"--site",
pghoard.test_site,
"--mode",
"restore",
"--output",
tmp_out,
"--xlog",
wal_seg,
])
with open(tmp_out, "rb") as fp:
restored_data = fp.read()
assert content == restored_data
def test_get_invalid(self, pghoard, tmpdir):
ne_wal_seg = "0000FFFF0000000C000000FE"
nonexistent_wal = "/{}/archive/{}".format(pghoard.test_site, ne_wal_seg)
# x-pghoard-target-path missing
conn = HTTPConnection(host="127.0.0.1", port=pghoard.config["http_port"])
conn.request("GET", nonexistent_wal)
status = conn.getresponse().status
assert status == 400
# missing WAL file
headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
conn.request("GET", nonexistent_wal, headers=headers)
status = conn.getresponse().status
assert status == 404
# no x-pghoard-target-path for head
headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
conn.request("HEAD", nonexistent_wal, headers=headers)
status = conn.getresponse().status
assert status == 400
# missing WAL file
headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
conn.request("HEAD", nonexistent_wal)
status = conn.getresponse().status
assert status == 404
# missing WAL file using restore_command
with pytest.raises(postgres_command.PGCError) as excinfo:
restore_command(
site=pghoard.test_site,
xlog=os.path.basename(nonexistent_wal),
host="127.0.0.1",
port=pghoard.config["http_port"],
output=None,
retry_interval=0.1
)
assert excinfo.value.exit_code == postgres_command.EXIT_NOT_FOUND
# write failures, this should be retried a couple of times
# start by making sure we can access the file normally
valid_wal_seg = "0000DDDD0000000D000000FC"
valid_wal = "/{}/xlog/{}".format(pghoard.test_site, valid_wal_seg)
store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
store.store_file_from_memory(valid_wal, wal_header_for_file(valid_wal_seg), metadata={"a": "b"})
conn.request("HEAD", valid_wal)
status = conn.getresponse().status
assert status == 200
restore_command(
site=pghoard.test_site,
xlog=os.path.basename(valid_wal),
host="127.0.0.1",
port=pghoard.config["http_port"],
output=None,
retry_interval=0.1
)
# write to non-existent directory
headers = {"x-pghoard-target-path": str(tmpdir.join("NA", "test_get_invalid"))}
conn.request("GET", valid_wal, headers=headers)
status = conn.getresponse().status
assert status == 409
def test_get_invalid(self, pghoard, tmpdir):
ne_wal_seg = "0000FFFF0000000C000000FE"
nonexistent_wal = "/{}/archive/{}".format(pghoard.test_site, ne_wal_seg)
# x-pghoard-target-path missing
conn = HTTPConnection(host="127.0.0.1", port=pghoard.config["http_port"])
conn.request("GET", nonexistent_wal)
status = conn.getresponse().status
assert status == 400
# missing WAL file
headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
conn.request("GET", nonexistent_wal, headers=headers)
status = conn.getresponse().status
assert status == 404
# no x-pghoard-target-path for head
headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
conn.request("HEAD", nonexistent_wal, headers=headers)
status = conn.getresponse().status
assert status == 400
# missing WAL file
headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
conn.request("HEAD", nonexistent_wal)
status = conn.getresponse().status
assert status == 404
# missing WAL file using restore_command
with pytest.raises(postgres_command.PGCError) as excinfo:
restore_command(site=pghoard.test_site, xlog=os.path.basename(nonexistent_wal),
host="127.0.0.1", port=pghoard.config["http_port"],
output=None, retry_interval=0.1)
assert excinfo.value.exit_code == postgres_command.EXIT_NOT_FOUND
# write failures, this should be retried a couple of times
# start by making sure we can access the file normally
valid_wal_seg = "0000DDDD0000000D000000FC"
valid_wal = "/{}/xlog/{}".format(pghoard.test_site, valid_wal_seg)
store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
store.store_file_from_memory(valid_wal, wal_header_for_file(valid_wal_seg), metadata={"a": "b"})
conn.request("HEAD", valid_wal)
status = conn.getresponse().status
assert status == 200
restore_command(site=pghoard.test_site, xlog=os.path.basename(valid_wal),
host="127.0.0.1", port=pghoard.config["http_port"],
output=None, retry_interval=0.1)
# write to non-existent directory
headers = {"x-pghoard-target-path": str(tmpdir.join("NA", "test_get_invalid"))}
conn.request("GET", valid_wal, headers=headers)
status = conn.getresponse().status
assert status == 409
def test_wal_fetch_optimization(self, pghoard):
# inject fake WAL and timeline files for testing
invalid_wal_name = "000000060000000000000001"
valid_wal_name = "000000060000000000000002"
wal_name_output = "optimization_output_filename"
output_path = os.path.join(
get_pg_wal_directory(
pghoard.config["backup_sites"][pghoard.test_site]),
wal_name_output)
invalid_wal_path = os.path.join(
get_pg_wal_directory(
pghoard.config["backup_sites"][pghoard.test_site]),
invalid_wal_name)
valid_wal_path = os.path.join(
get_pg_wal_directory(
pghoard.config["backup_sites"][pghoard.test_site]),
valid_wal_name)
with open(valid_wal_path, "wb") as out_file:
out_file.write(
wal_header_for_file(os.path.basename(valid_wal_path)))
with open(invalid_wal_path, "wb") as out_file:
# We use the wrong WAL file's name to generate the header on purpose to see that our check works
out_file.write(
wal_header_for_file(os.path.basename(valid_wal_path)))
restore_command(site=pghoard.test_site,
xlog=os.path.basename(valid_wal_name),
host="127.0.0.1",
port=pghoard.config["http_port"],
output=output_path,
retry_interval=0.1)
assert os.path.exists(output_path)
os.unlink(output_path)
with pytest.raises(postgres_command.PGCError):
restore_command(site=pghoard.test_site,
xlog=os.path.basename(invalid_wal_name),
host="127.0.0.1",
port=pghoard.config["http_port"],
output=output_path,
retry_interval=0.1)
assert not os.path.exists(output_path)
os.unlink(invalid_wal_path)
def test_get_archived_file(self, pghoard):
wal_seg_prev_tli = "00000001000000000000000F"
wal_seg = "00000002000000000000000F"
wal_file = "xlog/{}".format(wal_seg)
# NOTE: create WAL header for the "previous" timeline, this should be accepted
content = wal_header_for_file(wal_seg_prev_tli)
wal_dir = get_pg_wal_directory(pghoard.config["backup_sites"][pghoard.test_site])
archive_path = os.path.join(pghoard.test_site, wal_file)
compressor = pghoard.Compressor()
compressed_content = compressor.compress(content) + (compressor.flush() or b"")
metadata = {
"compression-algorithm": pghoard.config["compression"]["algorithm"],
"original-file-size": len(content),
}
store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
store.store_file_from_memory(archive_path, compressed_content, metadata=metadata)
restore_command(site=pghoard.test_site, xlog=wal_seg, output=None,
host="127.0.0.1", port=pghoard.config["http_port"])
restore_target = os.path.join(wal_dir, wal_seg)
restore_command(site=pghoard.test_site, xlog=wal_seg, output=restore_target,
host="127.0.0.1", port=pghoard.config["http_port"])
assert os.path.exists(restore_target) is True
with open(restore_target, "rb") as fp:
restored_data = fp.read()
assert content == restored_data
# test the same thing using restore as 'pghoard_postgres_command'
tmp_out = os.path.join(wal_dir, restore_target + ".cmd")
postgres_command.main([
"--host", "localhost",
"--port", str(pghoard.config["http_port"]),
"--site", pghoard.test_site,
"--mode", "restore",
"--output", tmp_out,
"--xlog", wal_seg,
])
with open(tmp_out, "rb") as fp:
restored_data = fp.read()
assert content == restored_data
def test_wal_fetch_optimization(self, pghoard):
# inject fake WAL and timeline files for testing
invalid_wal_name = "000000060000000000000001"
valid_wal_name = "000000060000000000000002"
wal_name_output = "optimization_output_filename"
output_path = os.path.join(
get_pg_wal_directory(pghoard.config["backup_sites"][pghoard.test_site]), wal_name_output)
invalid_wal_path = os.path.join(
get_pg_wal_directory(pghoard.config["backup_sites"][pghoard.test_site]), invalid_wal_name)
valid_wal_path = os.path.join(
get_pg_wal_directory(pghoard.config["backup_sites"][pghoard.test_site]), valid_wal_name)
with open(valid_wal_path, "wb") as out_file:
out_file.write(wal_header_for_file(os.path.basename(valid_wal_path)))
with open(invalid_wal_path, "wb") as out_file:
# We use the wrong WAL file's name to generate the header on purpose to see that our check works
out_file.write(wal_header_for_file(os.path.basename(valid_wal_path)))
restore_command(
site=pghoard.test_site,
xlog=os.path.basename(valid_wal_name),
host="127.0.0.1",
port=pghoard.config["http_port"],
output=output_path,
retry_interval=0.1)
assert os.path.exists(output_path)
os.unlink(output_path)
with pytest.raises(postgres_command.PGCError):
restore_command(
site=pghoard.test_site,
xlog=os.path.basename(invalid_wal_name),
host="127.0.0.1",
port=pghoard.config["http_port"],
output=output_path,
retry_interval=0.1)
assert not os.path.exists(output_path)
os.unlink(invalid_wal_path)
def test_restore_command_retry(self, pghoard):
failures = [0, ""]
orig_http_request = postgres_command.http_request
def fail_http_request(*args):
if failures[0] > 0:
failures[0] -= 1
raise socket.error(
"test_restore_command_retry failure: {}".format(
failures[1]))
return orig_http_request(*args)
postgres_command.http_request = fail_http_request
# create a valid WAL file and make sure we can restore it normally
wal_seg = "E" * 24
wal_path = "/{}/xlog/{}".format(pghoard.test_site, wal_seg)
store = pghoard.transfer_agents[0].get_object_storage(
pghoard.test_site)
store.store_file_from_memory(wal_path,
wal_header_for_file(wal_seg),
metadata={"a": "b"})
restore_command(site=pghoard.test_site,
xlog=wal_seg,
output=None,
host="127.0.0.1",
port=pghoard.config["http_port"],
retry_interval=0.1)
# now make the webserver fail all attempts
failures[0] = 4
failures[1] = "four fails"
# restore should fail
with pytest.raises(postgres_command.PGCError) as excinfo:
restore_command(site=pghoard.test_site,
xlog=wal_seg,
output=None,
host="127.0.0.1",
port=pghoard.config["http_port"],
retry_interval=0.1)
assert excinfo.value.exit_code == postgres_command.EXIT_ABORT
assert failures[
0] == 1 # fail_http_request should've have 1 failure left
# try with two failures, this should work on the third try
failures[0] = 2
failures[1] = "two fails"
restore_command(site=pghoard.test_site,
xlog=wal_seg,
output=None,
host="127.0.0.1",
port=pghoard.config["http_port"],
retry_interval=0.1)
assert failures[0] == 0
postgres_command.http_request = orig_http_request
def test_restore_command_retry(self, pghoard):
failures = [0, ""]
orig_http_request = postgres_command.http_request
def fail_http_request(*args):
if failures[0] > 0:
failures[0] -= 1
raise socket.error("test_restore_command_retry failure: {}".format(failures[1]))
return orig_http_request(*args)
postgres_command.http_request = fail_http_request
# create a valid WAL file and make sure we can restore it normally
wal_seg = "E" * 24
wal_path = "/{}/xlog/{}".format(pghoard.test_site, wal_seg)
store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
store.store_file_from_memory(wal_path, wal_header_for_file(wal_seg), metadata={"a": "b"})
restore_command(site=pghoard.test_site, xlog=wal_seg, output=None,
host="127.0.0.1", port=pghoard.config["http_port"],
retry_interval=0.1)
# now make the webserver fail all attempts
failures[0] = 4
failures[1] = "four fails"
# restore should fail
with pytest.raises(postgres_command.PGCError) as excinfo:
restore_command(site=pghoard.test_site, xlog=wal_seg, output=None,
host="127.0.0.1", port=pghoard.config["http_port"],
retry_interval=0.1)
assert excinfo.value.exit_code == postgres_command.EXIT_ABORT
assert failures[0] == 1 # fail_http_request should've have 1 failure left
# try with two failures, this should work on the third try
failures[0] = 2
failures[1] = "two fails"
restore_command(site=pghoard.test_site, xlog=wal_seg, output=None,
host="127.0.0.1", port=pghoard.config["http_port"],
retry_interval=0.1)
assert failures[0] == 0
postgres_command.http_request = orig_http_request
def test_get_invalid(self, pghoard, tmpdir):
ne_xlog_seg = "0000FFFF0000000C000000FE"
nonexistent_xlog = "/{}/archive/{}".format(pghoard.test_site, ne_xlog_seg)
# x-pghoard-target-path missing
conn = HTTPConnection(host="127.0.0.1", port=pghoard.config["http_port"])
conn.request("GET", nonexistent_xlog)
status = conn.getresponse().status
assert status == 400
# missing xlog file
headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
conn.request("GET", nonexistent_xlog, headers=headers)
status = conn.getresponse().status
assert status == 404
# no x-pghoard-target-path for head
headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
conn.request("HEAD", nonexistent_xlog, headers=headers)
status = conn.getresponse().status
assert status == 400
# missing xlog file
headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
conn.request("HEAD", nonexistent_xlog)
status = conn.getresponse().status
assert status == 404
# missing xlog file using restore_command
with pytest.raises(postgres_command.PGCError) as excinfo:
restore_command(site=pghoard.test_site, xlog=os.path.basename(nonexistent_xlog),
host="127.0.0.1", port=pghoard.config["http_port"],
output=None, retry_interval=0.1)
assert excinfo.value.exit_code == postgres_command.EXIT_NOT_FOUND
# write failures, this should be retried a couple of times
# start by making sure we can access the file normally
valid_xlog_seg = "0000DDDD0000000D000000FC"
valid_xlog = "/{}/xlog/{}".format(pghoard.test_site, valid_xlog_seg)
store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
store.store_file_from_memory(valid_xlog, wal_header_for_file(valid_xlog_seg), metadata={"a": "b"})
conn.request("HEAD", valid_xlog)
status = conn.getresponse().status
assert status == 200
restore_command(site=pghoard.test_site, xlog=os.path.basename(valid_xlog),
host="127.0.0.1", port=pghoard.config["http_port"],
output=None, retry_interval=0.1)
# write to non-existent directory
headers = {"x-pghoard-target-path": str(tmpdir.join("NA", "test_get_invalid"))}
conn.request("GET", valid_xlog, headers=headers)
status = conn.getresponse().status
assert status == 400
# inject a failure by making a static function fail
failures = [0, ""]
def get_failing_func(orig_func):
def failing_func(*args):
if failures[0] > 0:
failures[0] -= 1
raise Exception("test_get_invalid failure: {}".format(failures[1]))
return orig_func(*args)
return failing_func
for ta in pghoard.transfer_agents:
store = ta.get_object_storage(pghoard.test_site)
store.get_contents_to_string = get_failing_func(store.get_contents_to_string)
prefetch_n = pghoard.config["restore_prefetch"]
try:
# we should have two retries + all prefetch operations
pghoard.webserver.server.prefetch_404.clear()
failures[0] = 2 + prefetch_n
failures[1] = "test_two_fails_success"
headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid_2"))}
conn.request("GET", valid_xlog, headers=headers)
status = conn.getresponse().status
assert status == 201
assert failures[0] == 0
# so we should have a hard failure after three attempts
pghoard.webserver.server.prefetch_404.clear()
failures[0] = 4 + prefetch_n
failures[1] = "test_three_fails_error"
headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid_3"))}
conn.request("GET", valid_xlog, headers=headers)
status = conn.getresponse().status
assert status == 500
assert failures[0] == 1
finally:
# clear transfer cache to avoid using our failing versions
for ta in pghoard.transfer_agents:
ta.site_transfers = {}
def test_get_invalid(self, pghoard, tmpdir):
ne_xlog_seg = "0000FFFF0000000C000000FE"
nonexistent_xlog = "/{}/archive/{}".format(pghoard.test_site, ne_xlog_seg)
# x-pghoard-target-path missing
conn = HTTPConnection(host="127.0.0.1", port=pghoard.config["http_port"])
conn.request("GET", nonexistent_xlog)
status = conn.getresponse().status
assert status == 400
# missing xlog file
headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
conn.request("GET", nonexistent_xlog, headers=headers)
status = conn.getresponse().status
assert status == 404
# no x-pghoard-target-path for head
headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
conn.request("HEAD", nonexistent_xlog, headers=headers)
status = conn.getresponse().status
assert status == 400
# missing xlog file
headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
conn.request("HEAD", nonexistent_xlog)
status = conn.getresponse().status
assert status == 404
# missing xlog file using restore_command
with pytest.raises(postgres_command.PGCError) as excinfo:
restore_command(site=pghoard.test_site, xlog=os.path.basename(nonexistent_xlog),
host="127.0.0.1", port=pghoard.config["http_port"],
output=None, retry_interval=0.1)
assert excinfo.value.exit_code == postgres_command.EXIT_NOT_FOUND
# write failures, this should be retried a couple of times
# start by making sure we can access the file normally
valid_xlog_seg = "0000DDDD0000000D000000FC"
valid_xlog = "/{}/xlog/{}".format(pghoard.test_site, valid_xlog_seg)
store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
store.store_file_from_memory(valid_xlog, wal_header_for_file(valid_xlog_seg), metadata={"a": "b"})
conn.request("HEAD", valid_xlog)
status = conn.getresponse().status
assert status == 200
restore_command(site=pghoard.test_site, xlog=os.path.basename(valid_xlog),
host="127.0.0.1", port=pghoard.config["http_port"],
output=None, retry_interval=0.1)
# write to non-existent directory
headers = {"x-pghoard-target-path": str(tmpdir.join("NA", "test_get_invalid"))}
conn.request("GET", valid_xlog, headers=headers)
status = conn.getresponse().status
assert status == 400
# inject a failure by making a static function fail
failures = [0, ""]
def get_failing_func(orig_func):
def failing_func(*args):
if failures[0] > 0:
failures[0] -= 1
raise Exception("test_get_invalid failure: {}".format(failures[1]))
return orig_func(*args)
return failing_func
for ta in pghoard.transfer_agents:
store = ta.get_object_storage(pghoard.test_site)
store.get_contents_to_string = get_failing_func(store.get_contents_to_string)
prefetch_n = pghoard.config["restore_prefetch"]
try:
# we should have two retries + all prefetch operations
pghoard.webserver.server.prefetch_404.clear()
failures[0] = 2 + prefetch_n
failures[1] = "test_two_fails_success"
headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid_2"))}
conn.request("GET", valid_xlog, headers=headers)
status = conn.getresponse().status
assert status == 201
assert failures[0] == 0
# so we should have a hard failure after three attempts
pghoard.webserver.server.prefetch_404.clear()
failures[0] = 4 + prefetch_n
failures[1] = "test_three_fails_error"
headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid_3"))}
conn.request("GET", valid_xlog, headers=headers)
status = conn.getresponse().status
assert status == 500
assert failures[0] == 1
finally:
# clear transfer cache to avoid using our failing versions
for ta in pghoard.transfer_agents:
ta.site_transfers = {}
