def setup_method(self, method): super().setup_method(method) self.config = self.config_template({ "backup_sites": { self.test_site: { "object_storage": { "storage_type": "local", "directory": self.temp_dir }, }, }, }) self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog", "00000001000000000000000C") os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog")) with open(self.foo_path, "w") as out: out.write("foo") self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0", "base.tar.xz") os.makedirs(os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0")) with open(self.foo_basebackup_path, "w") as out: out.write("foo") self.compression_queue = Queue() self.transfer_queue = Queue() self.transfer_agent = TransferAgent( config=self.config, compression_queue=self.compression_queue, mp_manager=None, transfer_queue=self.transfer_queue, metrics=metrics.Metrics(statsd={}), shared_state_dict={} ) self.transfer_agent.start()
def setup_method(self, method): super().setup_method(method) self.config = self.config_template() self.config["backup_sites"][self.test_site]["object_storage"] = { "storage_type": "s3" } os.makedirs(self.config["alert_file_dir"], exist_ok=True) self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog", "00000001000000000000000C") os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog")) with open(self.foo_path, "w") as out: out.write("foo") self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0", "base.tar.xz") os.makedirs( os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0")) with open(self.foo_basebackup_path, "w") as out: out.write("foo") self.compression_queue = Queue() self.transfer_queue = Queue() self.transfer_agent = TransferAgent( config=self.config, compression_queue=self.compression_queue, transfer_queue=self.transfer_queue, stats=statsd.StatsClient(host=None), shared_state_dict={}) self.transfer_agent.start()
def setup_method(self, method): super().setup_method(method) self.config = self.config_template({ "backup_sites": { self.test_site: { "object_storage": { "storage_type": "local", "directory": self.temp_dir }, }, }, }) self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog", "00000001000000000000000C") os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog")) with open(self.foo_path, "w") as out: out.write("foo") self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0", "base.tar.xz") os.makedirs(os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0")) with open(self.foo_basebackup_path, "w") as out: out.write("foo") self.compression_queue = Queue() self.transfer_queue = Queue() self.transfer_agent = TransferAgent( config=self.config, compression_queue=self.compression_queue, mp_manager=None, transfer_queue=self.transfer_queue, metrics=metrics.Metrics(statsd={}), shared_state_dict={}) self.transfer_agent.start()
def setup_method(self, method): super().setup_method(method) self.config = { "backup_sites": { self.test_site: { "object_storage": { "storage_type": "s3", }, }, }, "backup_location": self.temp_dir, } self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog", "00000001000000000000000C") os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog")) with open(self.foo_path, "w") as out: out.write("foo") self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0", "base.tar.xz") os.makedirs(os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0")) with open(self.foo_basebackup_path, "w") as out: out.write("foo") self.compression_queue = Queue() self.transfer_queue = Queue() self.transfer_agent = TransferAgent(self.config, self.compression_queue, self.transfer_queue) self.transfer_agent.start()
def setup_method(self, method): super().setup_method(method) self.config = self.config_template() self.config["backup_sites"][self.test_site]["object_storage"] = {"storage_type": "s3"} os.makedirs(self.config["alert_file_dir"], exist_ok=True) self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog", "00000001000000000000000C") os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog")) with open(self.foo_path, "w") as out: out.write("foo") self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0", "base.tar.xz") os.makedirs(os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0")) with open(self.foo_basebackup_path, "w") as out: out.write("foo") self.compression_queue = Queue() self.transfer_queue = Queue() self.transfer_agent = TransferAgent( config=self.config, compression_queue=self.compression_queue, transfer_queue=self.transfer_queue, stats=statsd.StatsClient(host=None), shared_state_dict={}) self.transfer_agent.start()
def __init__(self, config_path): self.stats = None self.log = logging.getLogger("pghoard") self.log_level = None self.running = True self.config_path = config_path self.compression_queue = Queue() self.transfer_queue = Queue() self.syslog_handler = None self.config = {} self.site_transfers = {} self.state = { "backup_sites": {}, "startup_time": datetime.datetime.utcnow().isoformat(), } self.load_config() if not os.path.exists(self.config["backup_location"]): os.makedirs(self.config["backup_location"]) signal.signal(signal.SIGHUP, self.load_config) signal.signal(signal.SIGINT, self.quit) signal.signal(signal.SIGTERM, self.quit) self.time_of_last_backup = {} self.time_of_last_backup_check = {} self.basebackups = {} self.basebackups_callbacks = {} self.receivexlogs = {} self.compressors = [] self.walreceivers = {} self.transfer_agents = [] self.requested_basebackup_sites = set() self.inotify = InotifyWatcher(self.compression_queue) self.webserver = WebServer( self.config, self.requested_basebackup_sites, self.compression_queue, self.transfer_queue) for _ in range(self.config["compression"]["thread_count"]): compressor = CompressorThread( config_dict=self.config, compression_queue=self.compression_queue, transfer_queue=self.transfer_queue, stats=self.stats) self.compressors.append(compressor) compressor_state = {} # shared among transfer agents for _ in range(self.config["transfer"]["thread_count"]): ta = TransferAgent( config=self.config, compression_queue=self.compression_queue, transfer_queue=self.transfer_queue, stats=self.stats, shared_state_dict=compressor_state) self.transfer_agents.append(ta) logutil.notify_systemd("READY=1") self.log.info("pghoard initialized, own_hostname: %r, cwd: %r", socket.gethostname(), os.getcwd())
class TestTransferAgent(PGHoardTestCase): def setup_method(self, method): super().setup_method(method) self.config = self.config_template() self.config["backup_sites"][self.test_site]["object_storage"] = { "storage_type": "s3" } os.makedirs(self.config["alert_file_dir"], exist_ok=True) self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog", "00000001000000000000000C") os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog")) with open(self.foo_path, "w") as out: out.write("foo") self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0", "base.tar.xz") os.makedirs( os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0")) with open(self.foo_basebackup_path, "w") as out: out.write("foo") self.compression_queue = Queue() self.transfer_queue = Queue() self.transfer_agent = TransferAgent( config=self.config, compression_queue=self.compression_queue, transfer_queue=self.transfer_queue, stats=statsd.StatsClient(host=None), shared_state_dict={}) self.transfer_agent.start() def teardown_method(self, method): self.transfer_agent.running = False self.transfer_queue.put({"type": "QUIT"}) self.transfer_agent.join() super().teardown_method(method) def test_handle_download(self): callback_queue = Queue() self.transfer_agent.get_object_storage = MockStorage() self.transfer_queue.put({ "callback_queue": callback_queue, "filetype": "xlog", "local_path": self.temp_dir, "opaque": 42, "site": self.test_site, "target_path": self.temp_dir, "type": "DOWNLOAD", }) expected_event = { "blob": b"joo", "callback_queue": callback_queue, "local_path": self.temp_dir, "metadata": { "key": "value" }, "opaque": 42, "site": self.test_site, "type": "DECOMPRESSION", } assert self.compression_queue.get(timeout=1.0) == expected_event def test_handle_upload_xlog(self): callback_queue = Queue() storage = MockStorage() self.transfer_agent.get_object_storage = storage assert os.path.exists(self.foo_path) is True self.transfer_queue.put({ "callback_queue": callback_queue, "file_size": 3, "filetype": "xlog", "local_path": self.foo_path, "metadata": { "start-wal-segment": "00000001000000000000000C" }, "site": self.test_site, "type": "UPLOAD", }) assert callback_queue.get(timeout=1.0) == { "success": True, "opaque": None } assert os.path.exists(self.foo_path) is False def test_handle_upload_basebackup(self): callback_queue = Queue() storage = MockStorage() self.transfer_agent.get_object_storage = storage assert os.path.exists(self.foo_path) is True self.transfer_queue.put({ "callback_queue": callback_queue, "file_size": 3, "filetype": "basebackup", "local_path": self.foo_basebackup_path, "metadata": { "start-wal-segment": "00000001000000000000000C" }, "site": self.test_site, "type": "UPLOAD", }) assert callback_queue.get(timeout=1.0) == { "success": True, "opaque": None } assert os.path.exists(self.foo_basebackup_path) is False def test_handle_failing_upload_xlog(self): callback_queue = Queue() storage = MockStorageRaising() self.transfer_agent.get_object_storage = storage assert os.path.exists(self.foo_path) is True self.transfer_queue.put({ "callback_queue": callback_queue, "file_size": 3, "filetype": "xlog", "local_path": self.foo_path, "metadata": { "start-wal-segment": "00000001000000000000000C" }, "site": self.test_site, "type": "UPLOAD", }) with pytest.raises(Empty): callback_queue.get(timeout=3.0) alert_file_path = os.path.join(self.config["alert_file_dir"], "upload_retries_warning") assert os.path.exists(alert_file_path) is True os.unlink(alert_file_path)
def __init__(self, config_path): self.metrics = None self.log = logging.getLogger("pghoard") self.log_level = None self.running = True self.config_path = config_path self.compression_queue = Queue() self.transfer_queue = Queue() self.syslog_handler = None self.basebackups = {} self.basebackups_callbacks = {} self.receivexlogs = {} self.compressors = [] self.walreceivers = {} self.transfer_agents = [] self.config = {} self.mp_manager = None self.site_transfers = {} self.state = { "backup_sites": {}, "startup_time": datetime.datetime.utcnow().isoformat(), } self.transfer_agent_state = {} # shared among transfer agents self.load_config() if self.config["transfer"]["thread_count"] > 1: self.mp_manager = multiprocessing.Manager() if not os.path.exists(self.config["backup_location"]): os.makedirs(self.config["backup_location"]) # Read transfer_agent_state from state file if available so that there's no disruption # in the metrics we send out as a result of process restart state_file_path = self.config["json_state_file_path"] if os.path.exists(state_file_path): with open(state_file_path, "r") as fp: state = json.load(fp) self.transfer_agent_state = state.get( "transfer_agent_state") or {} signal.signal(signal.SIGHUP, self.load_config) signal.signal(signal.SIGINT, self.quit) signal.signal(signal.SIGTERM, self.quit) self.time_of_last_backup_check = {} self.requested_basebackup_sites = set() self.inotify = InotifyWatcher(self.compression_queue) self.webserver = WebServer(self.config, self.requested_basebackup_sites, self.compression_queue, self.transfer_queue, self.metrics) for _ in range(self.config["compression"]["thread_count"]): compressor = CompressorThread( config_dict=self.config, compression_queue=self.compression_queue, transfer_queue=self.transfer_queue, metrics=self.metrics) self.compressors.append(compressor) for _ in range(self.config["transfer"]["thread_count"]): ta = TransferAgent(config=self.config, compression_queue=self.compression_queue, mp_manager=self.mp_manager, transfer_queue=self.transfer_queue, metrics=self.metrics, shared_state_dict=self.transfer_agent_state) self.transfer_agents.append(ta) logutil.notify_systemd("READY=1") self.log.info("pghoard initialized, own_hostname: %r, cwd: %r", socket.gethostname(), os.getcwd())
class TestTransferAgent(PGHoardTestCase): def setup_method(self, method): super().setup_method(method) self.config = self.config_template({ "backup_sites": { self.test_site: { "object_storage": { "storage_type": "local", "directory": self.temp_dir }, }, }, }) self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog", "00000001000000000000000C") os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog")) with open(self.foo_path, "w") as out: out.write("foo") self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0", "base.tar.xz") os.makedirs(os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0")) with open(self.foo_basebackup_path, "w") as out: out.write("foo") self.compression_queue = Queue() self.transfer_queue = Queue() self.transfer_agent = TransferAgent( config=self.config, compression_queue=self.compression_queue, mp_manager=None, transfer_queue=self.transfer_queue, metrics=metrics.Metrics(statsd={}), shared_state_dict={}) self.transfer_agent.start() def teardown_method(self, method): self.transfer_agent.running = False self.transfer_queue.put({"type": "QUIT"}) self.transfer_agent.join() super().teardown_method(method) def test_handle_download(self): callback_queue = Queue() self.transfer_agent.get_object_storage = MockStorage() self.transfer_queue.put({ "callback_queue": callback_queue, "filetype": "xlog", "local_path": self.temp_dir, "opaque": 42, "site": self.test_site, "target_path": self.temp_dir, "type": "DOWNLOAD", }) event = callback_queue.get(timeout=1.0) assert event["success"] is False assert event["opaque"] == 42 assert isinstance(event["exception"], FileNotFoundFromStorageError) def test_handle_upload_xlog(self): callback_queue = Queue() storage = MockStorage() self.transfer_agent.get_object_storage = storage assert os.path.exists(self.foo_path) is True self.transfer_queue.put({ "callback_queue": callback_queue, "file_size": 3, "filetype": "xlog", "local_path": self.foo_path, "metadata": {"start-wal-segment": "00000001000000000000000C"}, "site": self.test_site, "type": "UPLOAD", }) assert callback_queue.get(timeout=1.0) == {"success": True, "opaque": None} assert os.path.exists(self.foo_path) is False def test_handle_upload_basebackup(self): callback_queue = Queue() storage = MockStorage() self.transfer_agent.get_object_storage = storage assert os.path.exists(self.foo_path) is True self.transfer_queue.put({ "callback_queue": callback_queue, "file_size": 3, "filetype": "basebackup", "local_path": self.foo_basebackup_path, "metadata": {"start-wal-segment": "00000001000000000000000C"}, "site": self.test_site, "type": "UPLOAD", }) assert callback_queue.get(timeout=1.0) == {"success": True, "opaque": None} assert os.path.exists(self.foo_basebackup_path) is False def test_handle_failing_upload_xlog(self): sleeps = [] def sleep(sleep_amount): sleeps.append(sleep_amount) time.sleep(0.001) callback_queue = Queue() storage = MockStorageRaising() self.transfer_agent.sleep = sleep self.transfer_agent.get_object_storage = storage assert os.path.exists(self.foo_path) is True self.transfer_queue.put({ "callback_queue": callback_queue, "file_size": 3, "filetype": "xlog", "local_path": self.foo_path, "metadata": {"start-wal-segment": "00000001000000000000000C"}, "site": self.test_site, "type": "UPLOAD", }) with pytest.raises(Empty): callback_queue.get(timeout=0.1) alert_file_path = os.path.join(self.config["alert_file_dir"], "upload_retries_warning") assert os.path.exists(alert_file_path) is True os.unlink(alert_file_path) expected_sleeps = [0.5, 1, 2, 4, 8, 16, 20, 20] assert sleeps[:8] == expected_sleeps
class TestTransferAgent(PGHoardTestCase): def setup_method(self, method): super().setup_method(method) self.config = self.config_template({ "backup_sites": { self.test_site: { "object_storage": { "storage_type": "local", "directory": self.temp_dir }, }, }, }) self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog", "00000001000000000000000C") os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog")) with open(self.foo_path, "w") as out: out.write("foo") self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0", "base.tar.xz") os.makedirs(os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0")) with open(self.foo_basebackup_path, "w") as out: out.write("foo") self.compression_queue = Queue() self.transfer_queue = Queue() self.transfer_agent = TransferAgent( config=self.config, compression_queue=self.compression_queue, mp_manager=None, transfer_queue=self.transfer_queue, metrics=metrics.Metrics(statsd={}), shared_state_dict={} ) self.transfer_agent.start() def teardown_method(self, method): self.transfer_agent.running = False self.transfer_queue.put({"type": "QUIT"}) self.transfer_agent.join() super().teardown_method(method) def test_handle_download(self): callback_queue = Queue() self.transfer_agent.get_object_storage = MockStorage() self.transfer_queue.put({ "callback_queue": callback_queue, "filetype": "xlog", "local_path": self.temp_dir, "opaque": 42, "site": self.test_site, "target_path": self.temp_dir, "type": "DOWNLOAD", }) event = callback_queue.get(timeout=1.0) assert event["success"] is False assert event["opaque"] == 42 assert isinstance(event["exception"], FileNotFoundFromStorageError) def test_handle_upload_xlog(self): callback_queue = Queue() storage = MockStorage() self.transfer_agent.get_object_storage = storage assert os.path.exists(self.foo_path) is True self.transfer_queue.put({ "callback_queue": callback_queue, "file_size": 3, "filetype": "xlog", "local_path": self.foo_path, "metadata": { "start-wal-segment": "00000001000000000000000C" }, "site": self.test_site, "type": "UPLOAD", }) assert callback_queue.get(timeout=1.0) == {"success": True, "opaque": None} assert os.path.exists(self.foo_path) is False def test_handle_upload_basebackup(self): callback_queue = Queue() storage = MockStorage() self.transfer_agent.get_object_storage = storage assert os.path.exists(self.foo_path) is True self.transfer_queue.put({ "callback_queue": callback_queue, "file_size": 3, "filetype": "basebackup", "local_path": self.foo_basebackup_path, "metadata": { "start-wal-segment": "00000001000000000000000C" }, "site": self.test_site, "type": "UPLOAD", }) assert callback_queue.get(timeout=1.0) == {"success": True, "opaque": None} assert os.path.exists(self.foo_basebackup_path) is False @pytest.mark.timeout(10) def test_handle_failing_upload_xlog(self): sleeps = [] def sleep(sleep_amount): sleeps.append(sleep_amount) time.sleep(0.001) callback_queue = Queue() storage = MockStorageRaising() self.transfer_agent.sleep = sleep self.transfer_agent.get_object_storage = storage assert os.path.exists(self.foo_path) is True self.transfer_queue.put({ "callback_queue": callback_queue, "file_size": 3, "filetype": "xlog", "local_path": self.foo_path, "metadata": { "start-wal-segment": "00000001000000000000000C" }, "site": self.test_site, "type": "UPLOAD", }) while len(sleeps) < 8: with pytest.raises(Empty): callback_queue.get(timeout=0.01) alert_file_path = os.path.join(self.config["alert_file_dir"], "upload_retries_warning") assert os.path.exists(alert_file_path) is True os.unlink(alert_file_path) expected_sleeps = [0.5, 1, 2, 4, 8, 16, 20, 20] assert sleeps[:8] == expected_sleeps
class TestTransferAgent(PGHoardTestCase): def setup_method(self, method): super().setup_method(method) self.config = self.config_template() self.config["backup_sites"][self.test_site]["object_storage"] = {"storage_type": "s3"} os.makedirs(self.config["alert_file_dir"], exist_ok=True) self.foo_path = os.path.join(self.temp_dir, self.test_site, "xlog", "00000001000000000000000C") os.makedirs(os.path.join(self.temp_dir, self.test_site, "xlog")) with open(self.foo_path, "w") as out: out.write("foo") self.foo_basebackup_path = os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0", "base.tar.xz") os.makedirs(os.path.join(self.temp_dir, self.test_site, "basebackup", "2015-04-15_0")) with open(self.foo_basebackup_path, "w") as out: out.write("foo") self.compression_queue = Queue() self.transfer_queue = Queue() self.transfer_agent = TransferAgent(self.config, self.compression_queue, self.transfer_queue) self.transfer_agent.start() def teardown_method(self, method): self.transfer_agent.running = False self.transfer_queue.put({"type": "QUIT"}) self.transfer_agent.join() super().teardown_method(method) def test_handle_download(self): callback_queue = Queue() self.transfer_agent.get_object_storage = MockStorage() self.transfer_queue.put({ "callback_queue": callback_queue, "filetype": "xlog", "local_path": self.temp_dir, "opaque": 42, "site": self.test_site, "target_path": self.temp_dir, "type": "DOWNLOAD", }) expected_event = { "blob": b"joo", "callback_queue": callback_queue, "local_path": self.temp_dir, "metadata": {"key": "value"}, "opaque": 42, "site": self.test_site, "type": "DECOMPRESSION", } assert self.compression_queue.get(timeout=1.0) == expected_event def test_handle_upload_xlog(self): callback_queue = Queue() storage = MockStorage() self.transfer_agent.get_object_storage = storage assert os.path.exists(self.foo_path) is True self.transfer_queue.put({ "callback_queue": callback_queue, "file_size": 3, "filetype": "xlog", "local_path": self.foo_path, "metadata": {"start-wal-segment": "00000001000000000000000C"}, "site": self.test_site, "type": "UPLOAD", }) assert callback_queue.get(timeout=1.0) == {"success": True, "opaque": None} assert os.path.exists(self.foo_path) is False def test_handle_upload_basebackup(self): callback_queue = Queue() storage = MockStorage() self.transfer_agent.get_object_storage = storage assert os.path.exists(self.foo_path) is True self.transfer_queue.put({ "callback_queue": callback_queue, "file_size": 3, "filetype": "basebackup", "local_path": self.foo_basebackup_path, "metadata": {"start-wal-segment": "00000001000000000000000C"}, "site": self.test_site, "type": "UPLOAD", }) assert callback_queue.get(timeout=1.0) == {"success": True, "opaque": None} assert os.path.exists(self.foo_basebackup_path) is False def test_handle_failing_upload_xlog(self): callback_queue = Queue() storage = MockStorageRaising() self.transfer_agent.get_object_storage = storage assert os.path.exists(self.foo_path) is True self.transfer_queue.put({ "callback_queue": callback_queue, "file_size": 3, "filetype": "xlog", "local_path": self.foo_path, "metadata": {"start-wal-segment": "00000001000000000000000C"}, "site": self.test_site, "type": "UPLOAD", }) with pytest.raises(Empty): callback_queue.get(timeout=3.0) alert_file_path = os.path.join(self.config["alert_file_dir"], "upload_retries_warning") assert os.path.exists(alert_file_path) is True os.unlink(alert_file_path)