Example #1
0
 def test_get_encrypted_archived_file(self, pghoard):
     xlog_seg = "000000090000000000000010"
     content = wal_header_for_file(xlog_seg)
     compressor = pghoard.Compressor()
     compressed_content = compressor.compress(content) + (compressor.flush() or b"")
     encryptor = Encryptor(CONSTANT_TEST_RSA_PUBLIC_KEY)
     encrypted_content = encryptor.update(compressed_content) + encryptor.finalize()
     pgdata = os.path.dirname(pghoard.config["backup_sites"][pghoard.test_site]["pg_xlog_directory"])
     archive_path = os.path.join(pghoard.test_site, "xlog", xlog_seg)
     metadata = {
         "compression-algorithm": pghoard.config["compression"]["algorithm"],
         "original-file-size": len(content),
         "encryption-key-id": "testkey",
     }
     store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
     store.store_file_from_memory(archive_path, encrypted_content, metadata=metadata)
     pghoard.webserver.config["backup_sites"][pghoard.test_site]["encryption_keys"] = {
         "testkey": {
             "public": CONSTANT_TEST_RSA_PUBLIC_KEY,
             "private": CONSTANT_TEST_RSA_PRIVATE_KEY,
         },
     }
     restore_target = os.path.join(pgdata, "pg_xlog", xlog_seg)
     restore_command(site=pghoard.test_site, xlog=xlog_seg, output=restore_target,
                     host="127.0.0.1", port=pghoard.config["http_port"])
     assert os.path.exists(restore_target)
     with open(restore_target, "rb") as fp:
         restored_data = fp.read()
     assert content == restored_data
Example #2
0
 def test_get_encrypted_archived_file(self, pghoard):
     wal_seg = "000000090000000000000010"
     content = wal_header_for_file(wal_seg)
     compressor = pghoard.Compressor()
     compressed_content = compressor.compress(content) + (compressor.flush() or b"")
     encryptor = Encryptor(CONSTANT_TEST_RSA_PUBLIC_KEY)
     encrypted_content = encryptor.update(compressed_content) + encryptor.finalize()
     wal_dir = get_pg_wal_directory(pghoard.config["backup_sites"][pghoard.test_site])
     archive_path = os.path.join(pghoard.test_site, "xlog", wal_seg)
     metadata = {
         "compression-algorithm": pghoard.config["compression"]["algorithm"],
         "original-file-size": len(content),
         "encryption-key-id": "testkey",
     }
     store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
     store.store_file_from_memory(archive_path, encrypted_content, metadata=metadata)
     pghoard.webserver.config["backup_sites"][pghoard.test_site]["encryption_keys"] = {
         "testkey": {
             "public": CONSTANT_TEST_RSA_PUBLIC_KEY,
             "private": CONSTANT_TEST_RSA_PRIVATE_KEY,
         },
     }
     restore_target = os.path.join(wal_dir, wal_seg)
     restore_command(site=pghoard.test_site, xlog=wal_seg, output=restore_target,
                     host="127.0.0.1", port=pghoard.config["http_port"])
     assert os.path.exists(restore_target)
     with open(restore_target, "rb") as fp:
         restored_data = fp.read()
     assert content == restored_data
Example #3
0
    def test_get_archived_file(self, pghoard):
        wal_seg_prev_tli = "00000001000000000000000F"
        wal_seg = "00000002000000000000000F"
        wal_file = "xlog/{}".format(wal_seg)
        # NOTE: create WAL header for the "previous" timeline, this should be accepted
        content = wal_header_for_file(wal_seg_prev_tli)
        wal_dir = get_pg_wal_directory(
            pghoard.config["backup_sites"][pghoard.test_site])
        archive_path = os.path.join(pghoard.test_site, wal_file)
        compressor = pghoard.Compressor()
        compressed_content = compressor.compress(content) + (
            compressor.flush() or b"")
        metadata = {
            "compression-algorithm":
            pghoard.config["compression"]["algorithm"],
            "original-file-size": len(content),
        }
        store = pghoard.transfer_agents[0].get_object_storage(
            pghoard.test_site)
        store.store_file_from_memory(archive_path,
                                     compressed_content,
                                     metadata=metadata)

        restore_command(site=pghoard.test_site,
                        xlog=wal_seg,
                        output=None,
                        host="127.0.0.1",
                        port=pghoard.config["http_port"])
        restore_target = os.path.join(wal_dir, wal_seg)

        restore_command(site=pghoard.test_site,
                        xlog=wal_seg,
                        output=restore_target,
                        host="127.0.0.1",
                        port=pghoard.config["http_port"])
        assert os.path.exists(restore_target) is True
        with open(restore_target, "rb") as fp:
            restored_data = fp.read()
        assert content == restored_data

        # test the same thing using restore as 'pghoard_postgres_command'
        tmp_out = os.path.join(wal_dir, restore_target + ".cmd")
        postgres_command.main([
            "--host",
            "localhost",
            "--port",
            str(pghoard.config["http_port"]),
            "--site",
            pghoard.test_site,
            "--mode",
            "restore",
            "--output",
            tmp_out,
            "--xlog",
            wal_seg,
        ])
        with open(tmp_out, "rb") as fp:
            restored_data = fp.read()
        assert content == restored_data
Example #4
0
    def test_get_invalid(self, pghoard, tmpdir):
        ne_wal_seg = "0000FFFF0000000C000000FE"
        nonexistent_wal = "/{}/archive/{}".format(pghoard.test_site, ne_wal_seg)
        # x-pghoard-target-path missing
        conn = HTTPConnection(host="127.0.0.1", port=pghoard.config["http_port"])
        conn.request("GET", nonexistent_wal)
        status = conn.getresponse().status
        assert status == 400
        # missing WAL file
        headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
        conn.request("GET", nonexistent_wal, headers=headers)
        status = conn.getresponse().status
        assert status == 404
        # no x-pghoard-target-path for head
        headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
        conn.request("HEAD", nonexistent_wal, headers=headers)
        status = conn.getresponse().status
        assert status == 400
        # missing WAL file
        headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
        conn.request("HEAD", nonexistent_wal)
        status = conn.getresponse().status
        assert status == 404
        # missing WAL file using restore_command
        with pytest.raises(postgres_command.PGCError) as excinfo:
            restore_command(
                site=pghoard.test_site,
                xlog=os.path.basename(nonexistent_wal),
                host="127.0.0.1",
                port=pghoard.config["http_port"],
                output=None,
                retry_interval=0.1
            )
        assert excinfo.value.exit_code == postgres_command.EXIT_NOT_FOUND

        # write failures, this should be retried a couple of times
        # start by making sure we can access the file normally
        valid_wal_seg = "0000DDDD0000000D000000FC"
        valid_wal = "/{}/xlog/{}".format(pghoard.test_site, valid_wal_seg)
        store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
        store.store_file_from_memory(valid_wal, wal_header_for_file(valid_wal_seg), metadata={"a": "b"})
        conn.request("HEAD", valid_wal)
        status = conn.getresponse().status
        assert status == 200
        restore_command(
            site=pghoard.test_site,
            xlog=os.path.basename(valid_wal),
            host="127.0.0.1",
            port=pghoard.config["http_port"],
            output=None,
            retry_interval=0.1
        )

        # write to non-existent directory
        headers = {"x-pghoard-target-path": str(tmpdir.join("NA", "test_get_invalid"))}
        conn.request("GET", valid_wal, headers=headers)
        status = conn.getresponse().status
        assert status == 409
Example #5
0
    def test_get_invalid(self, pghoard, tmpdir):
        ne_wal_seg = "0000FFFF0000000C000000FE"
        nonexistent_wal = "/{}/archive/{}".format(pghoard.test_site, ne_wal_seg)
        # x-pghoard-target-path missing
        conn = HTTPConnection(host="127.0.0.1", port=pghoard.config["http_port"])
        conn.request("GET", nonexistent_wal)
        status = conn.getresponse().status
        assert status == 400
        # missing WAL file
        headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
        conn.request("GET", nonexistent_wal, headers=headers)
        status = conn.getresponse().status
        assert status == 404
        # no x-pghoard-target-path for head
        headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
        conn.request("HEAD", nonexistent_wal, headers=headers)
        status = conn.getresponse().status
        assert status == 400
        # missing WAL file
        headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
        conn.request("HEAD", nonexistent_wal)
        status = conn.getresponse().status
        assert status == 404
        # missing WAL file using restore_command
        with pytest.raises(postgres_command.PGCError) as excinfo:
            restore_command(site=pghoard.test_site, xlog=os.path.basename(nonexistent_wal),
                            host="127.0.0.1", port=pghoard.config["http_port"],
                            output=None, retry_interval=0.1)
        assert excinfo.value.exit_code == postgres_command.EXIT_NOT_FOUND

        # write failures, this should be retried a couple of times
        # start by making sure we can access the file normally
        valid_wal_seg = "0000DDDD0000000D000000FC"
        valid_wal = "/{}/xlog/{}".format(pghoard.test_site, valid_wal_seg)
        store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
        store.store_file_from_memory(valid_wal, wal_header_for_file(valid_wal_seg), metadata={"a": "b"})
        conn.request("HEAD", valid_wal)
        status = conn.getresponse().status
        assert status == 200
        restore_command(site=pghoard.test_site, xlog=os.path.basename(valid_wal),
                        host="127.0.0.1", port=pghoard.config["http_port"],
                        output=None, retry_interval=0.1)

        # write to non-existent directory
        headers = {"x-pghoard-target-path": str(tmpdir.join("NA", "test_get_invalid"))}
        conn.request("GET", valid_wal, headers=headers)
        status = conn.getresponse().status
        assert status == 409
Example #6
0
    def test_wal_fetch_optimization(self, pghoard):
        # inject fake WAL and timeline files for testing
        invalid_wal_name = "000000060000000000000001"
        valid_wal_name = "000000060000000000000002"
        wal_name_output = "optimization_output_filename"
        output_path = os.path.join(
            get_pg_wal_directory(
                pghoard.config["backup_sites"][pghoard.test_site]),
            wal_name_output)
        invalid_wal_path = os.path.join(
            get_pg_wal_directory(
                pghoard.config["backup_sites"][pghoard.test_site]),
            invalid_wal_name)
        valid_wal_path = os.path.join(
            get_pg_wal_directory(
                pghoard.config["backup_sites"][pghoard.test_site]),
            valid_wal_name)

        with open(valid_wal_path, "wb") as out_file:
            out_file.write(
                wal_header_for_file(os.path.basename(valid_wal_path)))
        with open(invalid_wal_path, "wb") as out_file:
            # We use the wrong WAL file's name to generate the header on purpose to see that our check works
            out_file.write(
                wal_header_for_file(os.path.basename(valid_wal_path)))

        restore_command(site=pghoard.test_site,
                        xlog=os.path.basename(valid_wal_name),
                        host="127.0.0.1",
                        port=pghoard.config["http_port"],
                        output=output_path,
                        retry_interval=0.1)
        assert os.path.exists(output_path)
        os.unlink(output_path)

        with pytest.raises(postgres_command.PGCError):
            restore_command(site=pghoard.test_site,
                            xlog=os.path.basename(invalid_wal_name),
                            host="127.0.0.1",
                            port=pghoard.config["http_port"],
                            output=output_path,
                            retry_interval=0.1)
        assert not os.path.exists(output_path)
        os.unlink(invalid_wal_path)
Example #7
0
    def test_get_archived_file(self, pghoard):
        wal_seg_prev_tli = "00000001000000000000000F"
        wal_seg = "00000002000000000000000F"
        wal_file = "xlog/{}".format(wal_seg)
        # NOTE: create WAL header for the "previous" timeline, this should be accepted
        content = wal_header_for_file(wal_seg_prev_tli)
        wal_dir = get_pg_wal_directory(pghoard.config["backup_sites"][pghoard.test_site])
        archive_path = os.path.join(pghoard.test_site, wal_file)
        compressor = pghoard.Compressor()
        compressed_content = compressor.compress(content) + (compressor.flush() or b"")
        metadata = {
            "compression-algorithm": pghoard.config["compression"]["algorithm"],
            "original-file-size": len(content),
        }
        store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
        store.store_file_from_memory(archive_path, compressed_content, metadata=metadata)

        restore_command(site=pghoard.test_site, xlog=wal_seg, output=None,
                        host="127.0.0.1", port=pghoard.config["http_port"])
        restore_target = os.path.join(wal_dir, wal_seg)

        restore_command(site=pghoard.test_site, xlog=wal_seg, output=restore_target,
                        host="127.0.0.1", port=pghoard.config["http_port"])
        assert os.path.exists(restore_target) is True
        with open(restore_target, "rb") as fp:
            restored_data = fp.read()
        assert content == restored_data

        # test the same thing using restore as 'pghoard_postgres_command'
        tmp_out = os.path.join(wal_dir, restore_target + ".cmd")
        postgres_command.main([
            "--host", "localhost",
            "--port", str(pghoard.config["http_port"]),
            "--site", pghoard.test_site,
            "--mode", "restore",
            "--output", tmp_out,
            "--xlog", wal_seg,
        ])
        with open(tmp_out, "rb") as fp:
            restored_data = fp.read()
        assert content == restored_data
Example #8
0
    def test_wal_fetch_optimization(self, pghoard):
        # inject fake WAL and timeline files for testing
        invalid_wal_name = "000000060000000000000001"
        valid_wal_name = "000000060000000000000002"
        wal_name_output = "optimization_output_filename"
        output_path = os.path.join(
            get_pg_wal_directory(pghoard.config["backup_sites"][pghoard.test_site]), wal_name_output)
        invalid_wal_path = os.path.join(
            get_pg_wal_directory(pghoard.config["backup_sites"][pghoard.test_site]), invalid_wal_name)
        valid_wal_path = os.path.join(
            get_pg_wal_directory(pghoard.config["backup_sites"][pghoard.test_site]), valid_wal_name)

        with open(valid_wal_path, "wb") as out_file:
            out_file.write(wal_header_for_file(os.path.basename(valid_wal_path)))
        with open(invalid_wal_path, "wb") as out_file:
            # We use the wrong WAL file's name to generate the header on purpose to see that our check works
            out_file.write(wal_header_for_file(os.path.basename(valid_wal_path)))

        restore_command(
            site=pghoard.test_site,
            xlog=os.path.basename(valid_wal_name),
            host="127.0.0.1",
            port=pghoard.config["http_port"],
            output=output_path,
            retry_interval=0.1)
        assert os.path.exists(output_path)
        os.unlink(output_path)

        with pytest.raises(postgres_command.PGCError):
            restore_command(
                site=pghoard.test_site,
                xlog=os.path.basename(invalid_wal_name),
                host="127.0.0.1",
                port=pghoard.config["http_port"],
                output=output_path,
                retry_interval=0.1)
        assert not os.path.exists(output_path)
        os.unlink(invalid_wal_path)
Example #9
0
    def test_restore_command_retry(self, pghoard):
        failures = [0, ""]
        orig_http_request = postgres_command.http_request

        def fail_http_request(*args):
            if failures[0] > 0:
                failures[0] -= 1
                raise socket.error(
                    "test_restore_command_retry failure: {}".format(
                        failures[1]))
            return orig_http_request(*args)

        postgres_command.http_request = fail_http_request

        # create a valid WAL file and make sure we can restore it normally
        wal_seg = "E" * 24
        wal_path = "/{}/xlog/{}".format(pghoard.test_site, wal_seg)
        store = pghoard.transfer_agents[0].get_object_storage(
            pghoard.test_site)
        store.store_file_from_memory(wal_path,
                                     wal_header_for_file(wal_seg),
                                     metadata={"a": "b"})
        restore_command(site=pghoard.test_site,
                        xlog=wal_seg,
                        output=None,
                        host="127.0.0.1",
                        port=pghoard.config["http_port"],
                        retry_interval=0.1)

        # now make the webserver fail all attempts
        failures[0] = 4
        failures[1] = "four fails"
        # restore should fail
        with pytest.raises(postgres_command.PGCError) as excinfo:
            restore_command(site=pghoard.test_site,
                            xlog=wal_seg,
                            output=None,
                            host="127.0.0.1",
                            port=pghoard.config["http_port"],
                            retry_interval=0.1)
        assert excinfo.value.exit_code == postgres_command.EXIT_ABORT
        assert failures[
            0] == 1  # fail_http_request should've have 1 failure left

        # try with two failures, this should work on the third try
        failures[0] = 2
        failures[1] = "two fails"
        restore_command(site=pghoard.test_site,
                        xlog=wal_seg,
                        output=None,
                        host="127.0.0.1",
                        port=pghoard.config["http_port"],
                        retry_interval=0.1)
        assert failures[0] == 0

        postgres_command.http_request = orig_http_request
Example #10
0
    def test_restore_command_retry(self, pghoard):
        failures = [0, ""]
        orig_http_request = postgres_command.http_request

        def fail_http_request(*args):
            if failures[0] > 0:
                failures[0] -= 1
                raise socket.error("test_restore_command_retry failure: {}".format(failures[1]))
            return orig_http_request(*args)

        postgres_command.http_request = fail_http_request

        # create a valid WAL file and make sure we can restore it normally
        wal_seg = "E" * 24
        wal_path = "/{}/xlog/{}".format(pghoard.test_site, wal_seg)
        store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
        store.store_file_from_memory(wal_path, wal_header_for_file(wal_seg), metadata={"a": "b"})
        restore_command(site=pghoard.test_site, xlog=wal_seg, output=None,
                        host="127.0.0.1", port=pghoard.config["http_port"],
                        retry_interval=0.1)

        # now make the webserver fail all attempts
        failures[0] = 4
        failures[1] = "four fails"
        # restore should fail
        with pytest.raises(postgres_command.PGCError) as excinfo:
            restore_command(site=pghoard.test_site, xlog=wal_seg, output=None,
                            host="127.0.0.1", port=pghoard.config["http_port"],
                            retry_interval=0.1)
        assert excinfo.value.exit_code == postgres_command.EXIT_ABORT
        assert failures[0] == 1  # fail_http_request should've have 1 failure left

        # try with two failures, this should work on the third try
        failures[0] = 2
        failures[1] = "two fails"
        restore_command(site=pghoard.test_site, xlog=wal_seg, output=None,
                        host="127.0.0.1", port=pghoard.config["http_port"],
                        retry_interval=0.1)
        assert failures[0] == 0

        postgres_command.http_request = orig_http_request
Example #11
0
    def test_get_invalid(self, pghoard, tmpdir):
        ne_xlog_seg = "0000FFFF0000000C000000FE"
        nonexistent_xlog = "/{}/archive/{}".format(pghoard.test_site, ne_xlog_seg)
        # x-pghoard-target-path missing
        conn = HTTPConnection(host="127.0.0.1", port=pghoard.config["http_port"])
        conn.request("GET", nonexistent_xlog)
        status = conn.getresponse().status
        assert status == 400
        # missing xlog file
        headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
        conn.request("GET", nonexistent_xlog, headers=headers)
        status = conn.getresponse().status
        assert status == 404
        # no x-pghoard-target-path for head
        headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
        conn.request("HEAD", nonexistent_xlog, headers=headers)
        status = conn.getresponse().status
        assert status == 400
        # missing xlog file
        headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
        conn.request("HEAD", nonexistent_xlog)
        status = conn.getresponse().status
        assert status == 404
        # missing xlog file using restore_command
        with pytest.raises(postgres_command.PGCError) as excinfo:
            restore_command(site=pghoard.test_site, xlog=os.path.basename(nonexistent_xlog),
                            host="127.0.0.1", port=pghoard.config["http_port"],
                            output=None, retry_interval=0.1)
        assert excinfo.value.exit_code == postgres_command.EXIT_NOT_FOUND

        # write failures, this should be retried a couple of times
        # start by making sure we can access the file normally
        valid_xlog_seg = "0000DDDD0000000D000000FC"
        valid_xlog = "/{}/xlog/{}".format(pghoard.test_site, valid_xlog_seg)
        store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
        store.store_file_from_memory(valid_xlog, wal_header_for_file(valid_xlog_seg), metadata={"a": "b"})
        conn.request("HEAD", valid_xlog)
        status = conn.getresponse().status
        assert status == 200
        restore_command(site=pghoard.test_site, xlog=os.path.basename(valid_xlog),
                        host="127.0.0.1", port=pghoard.config["http_port"],
                        output=None, retry_interval=0.1)

        # write to non-existent directory
        headers = {"x-pghoard-target-path": str(tmpdir.join("NA", "test_get_invalid"))}
        conn.request("GET", valid_xlog, headers=headers)
        status = conn.getresponse().status
        assert status == 400

        # inject a failure by making a static function fail
        failures = [0, ""]

        def get_failing_func(orig_func):
            def failing_func(*args):
                if failures[0] > 0:
                    failures[0] -= 1
                    raise Exception("test_get_invalid failure: {}".format(failures[1]))
                return orig_func(*args)
            return failing_func

        for ta in pghoard.transfer_agents:
            store = ta.get_object_storage(pghoard.test_site)
            store.get_contents_to_string = get_failing_func(store.get_contents_to_string)

        prefetch_n = pghoard.config["restore_prefetch"]
        try:
            # we should have two retries + all prefetch operations
            pghoard.webserver.server.prefetch_404.clear()
            failures[0] = 2 + prefetch_n
            failures[1] = "test_two_fails_success"
            headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid_2"))}
            conn.request("GET", valid_xlog, headers=headers)
            status = conn.getresponse().status
            assert status == 201
            assert failures[0] == 0

            # so we should have a hard failure after three attempts
            pghoard.webserver.server.prefetch_404.clear()
            failures[0] = 4 + prefetch_n
            failures[1] = "test_three_fails_error"
            headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid_3"))}
            conn.request("GET", valid_xlog, headers=headers)
            status = conn.getresponse().status
            assert status == 500
            assert failures[0] == 1
        finally:
            # clear transfer cache to avoid using our failing versions
            for ta in pghoard.transfer_agents:
                ta.site_transfers = {}
Example #12
0
    def test_get_invalid(self, pghoard, tmpdir):
        ne_xlog_seg = "0000FFFF0000000C000000FE"
        nonexistent_xlog = "/{}/archive/{}".format(pghoard.test_site, ne_xlog_seg)
        # x-pghoard-target-path missing
        conn = HTTPConnection(host="127.0.0.1", port=pghoard.config["http_port"])
        conn.request("GET", nonexistent_xlog)
        status = conn.getresponse().status
        assert status == 400
        # missing xlog file
        headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
        conn.request("GET", nonexistent_xlog, headers=headers)
        status = conn.getresponse().status
        assert status == 404
        # no x-pghoard-target-path for head
        headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
        conn.request("HEAD", nonexistent_xlog, headers=headers)
        status = conn.getresponse().status
        assert status == 400
        # missing xlog file
        headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid"))}
        conn.request("HEAD", nonexistent_xlog)
        status = conn.getresponse().status
        assert status == 404
        # missing xlog file using restore_command
        with pytest.raises(postgres_command.PGCError) as excinfo:
            restore_command(site=pghoard.test_site, xlog=os.path.basename(nonexistent_xlog),
                            host="127.0.0.1", port=pghoard.config["http_port"],
                            output=None, retry_interval=0.1)
        assert excinfo.value.exit_code == postgres_command.EXIT_NOT_FOUND

        # write failures, this should be retried a couple of times
        # start by making sure we can access the file normally
        valid_xlog_seg = "0000DDDD0000000D000000FC"
        valid_xlog = "/{}/xlog/{}".format(pghoard.test_site, valid_xlog_seg)
        store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
        store.store_file_from_memory(valid_xlog, wal_header_for_file(valid_xlog_seg), metadata={"a": "b"})
        conn.request("HEAD", valid_xlog)
        status = conn.getresponse().status
        assert status == 200
        restore_command(site=pghoard.test_site, xlog=os.path.basename(valid_xlog),
                        host="127.0.0.1", port=pghoard.config["http_port"],
                        output=None, retry_interval=0.1)

        # write to non-existent directory
        headers = {"x-pghoard-target-path": str(tmpdir.join("NA", "test_get_invalid"))}
        conn.request("GET", valid_xlog, headers=headers)
        status = conn.getresponse().status
        assert status == 400

        # inject a failure by making a static function fail
        failures = [0, ""]

        def get_failing_func(orig_func):
            def failing_func(*args):
                if failures[0] > 0:
                    failures[0] -= 1
                    raise Exception("test_get_invalid failure: {}".format(failures[1]))
                return orig_func(*args)
            return failing_func

        for ta in pghoard.transfer_agents:
            store = ta.get_object_storage(pghoard.test_site)
            store.get_contents_to_string = get_failing_func(store.get_contents_to_string)

        prefetch_n = pghoard.config["restore_prefetch"]
        try:
            # we should have two retries + all prefetch operations
            pghoard.webserver.server.prefetch_404.clear()
            failures[0] = 2 + prefetch_n
            failures[1] = "test_two_fails_success"
            headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid_2"))}
            conn.request("GET", valid_xlog, headers=headers)
            status = conn.getresponse().status
            assert status == 201
            assert failures[0] == 0

            # so we should have a hard failure after three attempts
            pghoard.webserver.server.prefetch_404.clear()
            failures[0] = 4 + prefetch_n
            failures[1] = "test_three_fails_error"
            headers = {"x-pghoard-target-path": str(tmpdir.join("test_get_invalid_3"))}
            conn.request("GET", valid_xlog, headers=headers)
            status = conn.getresponse().status
            assert status == 500
            assert failures[0] == 1
        finally:
            # clear transfer cache to avoid using our failing versions
            for ta in pghoard.transfer_agents:
                ta.site_transfers = {}