def setup_method(self, method):
super().setup_method(method)
self.config = self.config_template({
"backup_sites": {
self.test_site: {
"object_storage": {
"storage_type": "local",
"directory": self.temp_dir
},
},
},
})
self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog", "00000001000000000000000C")
os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog"))
with open(self.foo_path, "w") as out:
out.write("foo")
self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0", "base.tar.xz")
os.makedirs(os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0"))
with open(self.foo_basebackup_path, "w") as out:
out.write("foo")
self.compression_queue = Queue()
self.transfer_queue = Queue()
self.transfer_agent = TransferAgent(
config=self.config,
compression_queue=self.compression_queue,
mp_manager=None,
transfer_queue=self.transfer_queue,
metrics=metrics.Metrics(statsd={}),
shared_state_dict={}
)
self.transfer_agent.start()
def setup_method(self, method):
super().setup_method(method)
self.config = self.config_template()
self.config["backup_sites"][self.test_site]["object_storage"] = {
"storage_type": "s3"
}
os.makedirs(self.config["alert_file_dir"], exist_ok=True)
self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog",
"00000001000000000000000C")
os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog"))
with open(self.foo_path, "w") as out:
out.write("foo")
self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site,
"basebackup", "2015-04-15_0",
"base.tar.xz")
os.makedirs(
os.path.join(self.temp_dir, self.test_site, "basebackup",
"2015-04-15_0"))
with open(self.foo_basebackup_path, "w") as out:
out.write("foo")
self.compression_queue = Queue()
self.transfer_queue = Queue()
self.transfer_agent = TransferAgent(
config=self.config,
compression_queue=self.compression_queue,
transfer_queue=self.transfer_queue,
stats=statsd.StatsClient(host=None),
shared_state_dict={})
self.transfer_agent.start()
def setup_method(self, method):
super().setup_method(method)
self.config = self.config_template({
"backup_sites": {
self.test_site: {
"object_storage": {
"storage_type": "local",
"directory": self.temp_dir
},
},
},
})
self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog", "00000001000000000000000C")
os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog"))
with open(self.foo_path, "w") as out:
out.write("foo")
self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0", "base.tar.xz")
os.makedirs(os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0"))
with open(self.foo_basebackup_path, "w") as out:
out.write("foo")
self.compression_queue = Queue()
self.transfer_queue = Queue()
self.transfer_agent = TransferAgent(
config=self.config,
compression_queue=self.compression_queue,
mp_manager=None,
transfer_queue=self.transfer_queue,
metrics=metrics.Metrics(statsd={}),
shared_state_dict={})
self.transfer_agent.start()
def setup_method(self, method):
super().setup_method(method)
self.config = {
"backup_sites": {
self.test_site: {
"object_storage": {
"storage_type": "s3",
},
},
},
"backup_location": self.temp_dir,
}
self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog", "00000001000000000000000C")
os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog"))
with open(self.foo_path, "w") as out:
out.write("foo")
self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0", "base.tar.xz")
os.makedirs(os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0"))
with open(self.foo_basebackup_path, "w") as out:
out.write("foo")
self.compression_queue = Queue()
self.transfer_queue = Queue()
self.transfer_agent = TransferAgent(self.config, self.compression_queue, self.transfer_queue)
self.transfer_agent.start()
def setup_method(self, method):
super().setup_method(method)
self.config = self.config_template()
self.config["backup_sites"][self.test_site]["object_storage"] = {"storage_type": "s3"}
os.makedirs(self.config["alert_file_dir"], exist_ok=True)
self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog", "00000001000000000000000C")
os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog"))
with open(self.foo_path, "w") as out:
out.write("foo")
self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0", "base.tar.xz")
os.makedirs(os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0"))
with open(self.foo_basebackup_path, "w") as out:
out.write("foo")
self.compression_queue = Queue()
self.transfer_queue = Queue()
self.transfer_agent = TransferAgent(
config=self.config,
compression_queue=self.compression_queue,
transfer_queue=self.transfer_queue,
stats=statsd.StatsClient(host=None),
shared_state_dict={})
self.transfer_agent.start()
def __init__(self, config_path):
self.stats = None
self.log = logging.getLogger("pghoard")
self.log_level = None
self.running = True
self.config_path = config_path
self.compression_queue = Queue()
self.transfer_queue = Queue()
self.syslog_handler = None
self.config = {}
self.site_transfers = {}
self.state = {
"backup_sites": {},
"startup_time": datetime.datetime.utcnow().isoformat(),
}
self.load_config()
if not os.path.exists(self.config["backup_location"]):
os.makedirs(self.config["backup_location"])
signal.signal(signal.SIGHUP, self.load_config)
signal.signal(signal.SIGINT, self.quit)
signal.signal(signal.SIGTERM, self.quit)
self.time_of_last_backup = {}
self.time_of_last_backup_check = {}
self.basebackups = {}
self.basebackups_callbacks = {}
self.receivexlogs = {}
self.compressors = []
self.walreceivers = {}
self.transfer_agents = []
self.requested_basebackup_sites = set()
self.inotify = InotifyWatcher(self.compression_queue)
self.webserver = WebServer(
self.config,
self.requested_basebackup_sites,
self.compression_queue,
self.transfer_queue)
for _ in range(self.config["compression"]["thread_count"]):
compressor = CompressorThread(
config_dict=self.config,
compression_queue=self.compression_queue,
transfer_queue=self.transfer_queue,
stats=self.stats)
self.compressors.append(compressor)
compressor_state = {} # shared among transfer agents
for _ in range(self.config["transfer"]["thread_count"]):
ta = TransferAgent(
config=self.config,
compression_queue=self.compression_queue,
transfer_queue=self.transfer_queue,
stats=self.stats,
shared_state_dict=compressor_state)
self.transfer_agents.append(ta)
logutil.notify_systemd("READY=1")
self.log.info("pghoard initialized, own_hostname: %r, cwd: %r", socket.gethostname(), os.getcwd())
class TestTransferAgent(PGHoardTestCase):
def setup_method(self, method):
super().setup_method(method)
self.config = self.config_template()
self.config["backup_sites"][self.test_site]["object_storage"] = {
"storage_type": "s3"
}
os.makedirs(self.config["alert_file_dir"], exist_ok=True)
self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog",
"00000001000000000000000C")
os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog"))
with open(self.foo_path, "w") as out:
out.write("foo")
self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site,
"basebackup", "2015-04-15_0",
"base.tar.xz")
os.makedirs(
os.path.join(self.temp_dir, self.test_site, "basebackup",
"2015-04-15_0"))
with open(self.foo_basebackup_path, "w") as out:
out.write("foo")
self.compression_queue = Queue()
self.transfer_queue = Queue()
self.transfer_agent = TransferAgent(
config=self.config,
compression_queue=self.compression_queue,
transfer_queue=self.transfer_queue,
stats=statsd.StatsClient(host=None),
shared_state_dict={})
self.transfer_agent.start()
def teardown_method(self, method):
self.transfer_agent.running = False
self.transfer_queue.put({"type": "QUIT"})
self.transfer_agent.join()
super().teardown_method(method)
def test_handle_download(self):
callback_queue = Queue()
self.transfer_agent.get_object_storage = MockStorage()
self.transfer_queue.put({
"callback_queue": callback_queue,
"filetype": "xlog",
"local_path": self.temp_dir,
"opaque": 42,
"site": self.test_site,
"target_path": self.temp_dir,
"type": "DOWNLOAD",
})
expected_event = {
"blob": b"joo",
"callback_queue": callback_queue,
"local_path": self.temp_dir,
"metadata": {
"key": "value"
},
"opaque": 42,
"site": self.test_site,
"type": "DECOMPRESSION",
}
assert self.compression_queue.get(timeout=1.0) == expected_event
def test_handle_upload_xlog(self):
callback_queue = Queue()
storage = MockStorage()
self.transfer_agent.get_object_storage = storage
assert os.path.exists(self.foo_path) is True
self.transfer_queue.put({
"callback_queue": callback_queue,
"file_size": 3,
"filetype": "xlog",
"local_path": self.foo_path,
"metadata": {
"start-wal-segment": "00000001000000000000000C"
},
"site": self.test_site,
"type": "UPLOAD",
})
assert callback_queue.get(timeout=1.0) == {
"success": True,
"opaque": None
}
assert os.path.exists(self.foo_path) is False
def test_handle_upload_basebackup(self):
callback_queue = Queue()
storage = MockStorage()
self.transfer_agent.get_object_storage = storage
assert os.path.exists(self.foo_path) is True
self.transfer_queue.put({
"callback_queue": callback_queue,
"file_size": 3,
"filetype": "basebackup",
"local_path": self.foo_basebackup_path,
"metadata": {
"start-wal-segment": "00000001000000000000000C"
},
"site": self.test_site,
"type": "UPLOAD",
})
assert callback_queue.get(timeout=1.0) == {
"success": True,
"opaque": None
}
assert os.path.exists(self.foo_basebackup_path) is False
def test_handle_failing_upload_xlog(self):
callback_queue = Queue()
storage = MockStorageRaising()
self.transfer_agent.get_object_storage = storage
assert os.path.exists(self.foo_path) is True
self.transfer_queue.put({
"callback_queue": callback_queue,
"file_size": 3,
"filetype": "xlog",
"local_path": self.foo_path,
"metadata": {
"start-wal-segment": "00000001000000000000000C"
},
"site": self.test_site,
"type": "UPLOAD",
})
with pytest.raises(Empty):
callback_queue.get(timeout=3.0)
alert_file_path = os.path.join(self.config["alert_file_dir"],
"upload_retries_warning")
assert os.path.exists(alert_file_path) is True
os.unlink(alert_file_path)
def __init__(self, config_path):
self.metrics = None
self.log = logging.getLogger("pghoard")
self.log_level = None
self.running = True
self.config_path = config_path
self.compression_queue = Queue()
self.transfer_queue = Queue()
self.syslog_handler = None
self.basebackups = {}
self.basebackups_callbacks = {}
self.receivexlogs = {}
self.compressors = []
self.walreceivers = {}
self.transfer_agents = []
self.config = {}
self.mp_manager = None
self.site_transfers = {}
self.state = {
"backup_sites": {},
"startup_time": datetime.datetime.utcnow().isoformat(),
}
self.transfer_agent_state = {} # shared among transfer agents
self.load_config()
if self.config["transfer"]["thread_count"] > 1:
self.mp_manager = multiprocessing.Manager()
if not os.path.exists(self.config["backup_location"]):
os.makedirs(self.config["backup_location"])
# Read transfer_agent_state from state file if available so that there's no disruption
# in the metrics we send out as a result of process restart
state_file_path = self.config["json_state_file_path"]
if os.path.exists(state_file_path):
with open(state_file_path, "r") as fp:
state = json.load(fp)
self.transfer_agent_state = state.get(
"transfer_agent_state") or {}
signal.signal(signal.SIGHUP, self.load_config)
signal.signal(signal.SIGINT, self.quit)
signal.signal(signal.SIGTERM, self.quit)
self.time_of_last_backup_check = {}
self.requested_basebackup_sites = set()
self.inotify = InotifyWatcher(self.compression_queue)
self.webserver = WebServer(self.config,
self.requested_basebackup_sites,
self.compression_queue, self.transfer_queue,
self.metrics)
for _ in range(self.config["compression"]["thread_count"]):
compressor = CompressorThread(
config_dict=self.config,
compression_queue=self.compression_queue,
transfer_queue=self.transfer_queue,
metrics=self.metrics)
self.compressors.append(compressor)
for _ in range(self.config["transfer"]["thread_count"]):
ta = TransferAgent(config=self.config,
compression_queue=self.compression_queue,
mp_manager=self.mp_manager,
transfer_queue=self.transfer_queue,
metrics=self.metrics,
shared_state_dict=self.transfer_agent_state)
self.transfer_agents.append(ta)
logutil.notify_systemd("READY=1")
self.log.info("pghoard initialized, own_hostname: %r, cwd: %r",
socket.gethostname(), os.getcwd())
class TestTransferAgent(PGHoardTestCase):
def setup_method(self, method):
super().setup_method(method)
self.config = self.config_template({
"backup_sites": {
self.test_site: {
"object_storage": {
"storage_type": "local",
"directory": self.temp_dir
},
},
},
})
self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog", "00000001000000000000000C")
os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog"))
with open(self.foo_path, "w") as out:
out.write("foo")
self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0", "base.tar.xz")
os.makedirs(os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0"))
with open(self.foo_basebackup_path, "w") as out:
out.write("foo")
self.compression_queue = Queue()
self.transfer_queue = Queue()
self.transfer_agent = TransferAgent(
config=self.config,
compression_queue=self.compression_queue,
mp_manager=None,
transfer_queue=self.transfer_queue,
metrics=metrics.Metrics(statsd={}),
shared_state_dict={})
self.transfer_agent.start()
def teardown_method(self, method):
self.transfer_agent.running = False
self.transfer_queue.put({"type": "QUIT"})
self.transfer_agent.join()
super().teardown_method(method)
def test_handle_download(self):
callback_queue = Queue()
self.transfer_agent.get_object_storage = MockStorage()
self.transfer_queue.put({
"callback_queue": callback_queue,
"filetype": "xlog",
"local_path": self.temp_dir,
"opaque": 42,
"site": self.test_site,
"target_path": self.temp_dir,
"type": "DOWNLOAD",
})
event = callback_queue.get(timeout=1.0)
assert event["success"] is False
assert event["opaque"] == 42
assert isinstance(event["exception"], FileNotFoundFromStorageError)
def test_handle_upload_xlog(self):
callback_queue = Queue()
storage = MockStorage()
self.transfer_agent.get_object_storage = storage
assert os.path.exists(self.foo_path) is True
self.transfer_queue.put({
"callback_queue": callback_queue,
"file_size": 3,
"filetype": "xlog",
"local_path": self.foo_path,
"metadata": {"start-wal-segment": "00000001000000000000000C"},
"site": self.test_site,
"type": "UPLOAD",
})
assert callback_queue.get(timeout=1.0) == {"success": True, "opaque": None}
assert os.path.exists(self.foo_path) is False
def test_handle_upload_basebackup(self):
callback_queue = Queue()
storage = MockStorage()
self.transfer_agent.get_object_storage = storage
assert os.path.exists(self.foo_path) is True
self.transfer_queue.put({
"callback_queue": callback_queue,
"file_size": 3,
"filetype": "basebackup",
"local_path": self.foo_basebackup_path,
"metadata": {"start-wal-segment": "00000001000000000000000C"},
"site": self.test_site,
"type": "UPLOAD",
})
assert callback_queue.get(timeout=1.0) == {"success": True, "opaque": None}
assert os.path.exists(self.foo_basebackup_path) is False
def test_handle_failing_upload_xlog(self):
sleeps = []
def sleep(sleep_amount):
sleeps.append(sleep_amount)
time.sleep(0.001)
callback_queue = Queue()
storage = MockStorageRaising()
self.transfer_agent.sleep = sleep
self.transfer_agent.get_object_storage = storage
assert os.path.exists(self.foo_path) is True
self.transfer_queue.put({
"callback_queue": callback_queue,
"file_size": 3,
"filetype": "xlog",
"local_path": self.foo_path,
"metadata": {"start-wal-segment": "00000001000000000000000C"},
"site": self.test_site,
"type": "UPLOAD",
})
with pytest.raises(Empty):
callback_queue.get(timeout=0.1)
alert_file_path = os.path.join(self.config["alert_file_dir"], "upload_retries_warning")
assert os.path.exists(alert_file_path) is True
os.unlink(alert_file_path)
expected_sleeps = [0.5, 1, 2, 4, 8, 16, 20, 20]
assert sleeps[:8] == expected_sleeps
class TestTransferAgent(PGHoardTestCase):
def setup_method(self, method):
super().setup_method(method)
self.config = self.config_template({
"backup_sites": {
self.test_site: {
"object_storage": {
"storage_type": "local",
"directory": self.temp_dir
},
},
},
})
self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog", "00000001000000000000000C")
os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog"))
with open(self.foo_path, "w") as out:
out.write("foo")
self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0", "base.tar.xz")
os.makedirs(os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0"))
with open(self.foo_basebackup_path, "w") as out:
out.write("foo")
self.compression_queue = Queue()
self.transfer_queue = Queue()
self.transfer_agent = TransferAgent(
config=self.config,
compression_queue=self.compression_queue,
mp_manager=None,
transfer_queue=self.transfer_queue,
metrics=metrics.Metrics(statsd={}),
shared_state_dict={}
)
self.transfer_agent.start()
def teardown_method(self, method):
self.transfer_agent.running = False
self.transfer_queue.put({"type": "QUIT"})
self.transfer_agent.join()
super().teardown_method(method)
def test_handle_download(self):
callback_queue = Queue()
self.transfer_agent.get_object_storage = MockStorage()
self.transfer_queue.put({
"callback_queue": callback_queue,
"filetype": "xlog",
"local_path": self.temp_dir,
"opaque": 42,
"site": self.test_site,
"target_path": self.temp_dir,
"type": "DOWNLOAD",
})
event = callback_queue.get(timeout=1.0)
assert event["success"] is False
assert event["opaque"] == 42
assert isinstance(event["exception"], FileNotFoundFromStorageError)
def test_handle_upload_xlog(self):
callback_queue = Queue()
storage = MockStorage()
self.transfer_agent.get_object_storage = storage
assert os.path.exists(self.foo_path) is True
self.transfer_queue.put({
"callback_queue": callback_queue,
"file_size": 3,
"filetype": "xlog",
"local_path": self.foo_path,
"metadata": {
"start-wal-segment": "00000001000000000000000C"
},
"site": self.test_site,
"type": "UPLOAD",
})
assert callback_queue.get(timeout=1.0) == {"success": True, "opaque": None}
assert os.path.exists(self.foo_path) is False
def test_handle_upload_basebackup(self):
callback_queue = Queue()
storage = MockStorage()
self.transfer_agent.get_object_storage = storage
assert os.path.exists(self.foo_path) is True
self.transfer_queue.put({
"callback_queue": callback_queue,
"file_size": 3,
"filetype": "basebackup",
"local_path": self.foo_basebackup_path,
"metadata": {
"start-wal-segment": "00000001000000000000000C"
},
"site": self.test_site,
"type": "UPLOAD",
})
assert callback_queue.get(timeout=1.0) == {"success": True, "opaque": None}
assert os.path.exists(self.foo_basebackup_path) is False
@pytest.mark.timeout(10)
def test_handle_failing_upload_xlog(self):
sleeps = []
def sleep(sleep_amount):
sleeps.append(sleep_amount)
time.sleep(0.001)
callback_queue = Queue()
storage = MockStorageRaising()
self.transfer_agent.sleep = sleep
self.transfer_agent.get_object_storage = storage
assert os.path.exists(self.foo_path) is True
self.transfer_queue.put({
"callback_queue": callback_queue,
"file_size": 3,
"filetype": "xlog",
"local_path": self.foo_path,
"metadata": {
"start-wal-segment": "00000001000000000000000C"
},
"site": self.test_site,
"type": "UPLOAD",
})
while len(sleeps) < 8:
with pytest.raises(Empty):
callback_queue.get(timeout=0.01)
alert_file_path = os.path.join(self.config["alert_file_dir"], "upload_retries_warning")
assert os.path.exists(alert_file_path) is True
os.unlink(alert_file_path)
expected_sleeps = [0.5, 1, 2, 4, 8, 16, 20, 20]
assert sleeps[:8] == expected_sleeps
class TestTransferAgent(PGHoardTestCase):
def setup_method(self, method):
super().setup_method(method)
self.config = self.config_template()
self.config["backup_sites"][self.test_site]["object_storage"] = {"storage_type": "s3"}
os.makedirs(self.config["alert_file_dir"], exist_ok=True)
self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog", "00000001000000000000000C")
os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog"))
with open(self.foo_path, "w") as out:
out.write("foo")
self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0", "base.tar.xz")
os.makedirs(os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0"))
with open(self.foo_basebackup_path, "w") as out:
out.write("foo")
self.compression_queue = Queue()
self.transfer_queue = Queue()
self.transfer_agent = TransferAgent(self.config, self.compression_queue, self.transfer_queue)
self.transfer_agent.start()
def teardown_method(self, method):
self.transfer_agent.running = False
self.transfer_queue.put({"type": "QUIT"})
self.transfer_agent.join()
super().teardown_method(method)
def test_handle_download(self):
callback_queue = Queue()
self.transfer_agent.get_object_storage = MockStorage()
self.transfer_queue.put({
"callback_queue": callback_queue,
"filetype": "xlog",
"local_path": self.temp_dir,
"opaque": 42,
"site": self.test_site,
"target_path": self.temp_dir,
"type": "DOWNLOAD",
})
expected_event = {
"blob": b"joo",
"callback_queue": callback_queue,
"local_path": self.temp_dir,
"metadata": {"key": "value"},
"opaque": 42,
"site": self.test_site,
"type": "DECOMPRESSION",
}
assert self.compression_queue.get(timeout=1.0) == expected_event
def test_handle_upload_xlog(self):
callback_queue = Queue()
storage = MockStorage()
self.transfer_agent.get_object_storage = storage
assert os.path.exists(self.foo_path) is True
self.transfer_queue.put({
"callback_queue": callback_queue,
"file_size": 3,
"filetype": "xlog",
"local_path": self.foo_path,
"metadata": {"start-wal-segment": "00000001000000000000000C"},
"site": self.test_site,
"type": "UPLOAD",
})
assert callback_queue.get(timeout=1.0) == {"success": True, "opaque": None}
assert os.path.exists(self.foo_path) is False
def test_handle_upload_basebackup(self):
callback_queue = Queue()
storage = MockStorage()
self.transfer_agent.get_object_storage = storage
assert os.path.exists(self.foo_path) is True
self.transfer_queue.put({
"callback_queue": callback_queue,
"file_size": 3,
"filetype": "basebackup",
"local_path": self.foo_basebackup_path,
"metadata": {"start-wal-segment": "00000001000000000000000C"},
"site": self.test_site,
"type": "UPLOAD",
})
assert callback_queue.get(timeout=1.0) == {"success": True, "opaque": None}
assert os.path.exists(self.foo_basebackup_path) is False
def test_handle_failing_upload_xlog(self):
callback_queue = Queue()
storage = MockStorageRaising()
self.transfer_agent.get_object_storage = storage
assert os.path.exists(self.foo_path) is True
self.transfer_queue.put({
"callback_queue": callback_queue,
"file_size": 3,
"filetype": "xlog",
"local_path": self.foo_path,
"metadata": {"start-wal-segment": "00000001000000000000000C"},
"site": self.test_site,
"type": "UPLOAD",
})
with pytest.raises(Empty):
callback_queue.get(timeout=3.0)
alert_file_path = os.path.join(self.config["alert_file_dir"], "upload_retries_warning")
assert os.path.exists(alert_file_path) is True
os.unlink(alert_file_path)
