def _verify_wal(self, filetype, filename, path):
if filetype != "xlog":
return
try:
wal.verify_wal(wal_name=filename, filepath=path)
except ValueError as ex:
raise HttpResponse(str(ex), status=412)
def _verify_wal(self, filetype, filename, path):
if filetype != "xlog":
return
try:
wal.verify_wal(wal_name=filename, filepath=path)
except ValueError as ex:
raise HttpResponse(str(ex), status=412)
def _create_prefetch_operations(self, site, filetype, filename):
if filetype not in {"timeline", "xlog"}:
return
prefetch_n = self.server.config["restore_prefetch"]
if prefetch_n <= 0:
return
xlog_dir = get_pg_wal_directory(
self.server.config["backup_sites"][site])
names = []
if filetype == "timeline":
tli_num = int(filename.replace(".history", ""), 16)
for _ in range(prefetch_n):
tli_num += 1
prefetch_name = "{:08X}.history".format(tli_num)
if os.path.isfile(os.path.join(xlog_dir, prefetch_name)):
continue
names.append(prefetch_name)
elif filetype == "xlog":
xlog_num = int(filename, 16)
for _ in range(prefetch_n):
if xlog_num & 0xFF == 0xFF:
xlog_num += 0xFFFFFF00
xlog_num += 1
prefetch_name = "{:024X}".format(xlog_num)
xlog_path = os.path.join(xlog_dir, prefetch_name)
if os.path.isfile(xlog_path):
try:
wal.verify_wal(wal_name=prefetch_name,
filepath=xlog_path)
continue
except ValueError as e:
self.server.log.debug(
"(Prefetch) File %s already exists but is invalid: %r",
xlog_path, e)
names.append(prefetch_name)
for obname in names:
key = self._make_file_key(site, filetype, obname)
if key in self.server.prefetch_404:
continue # previously failed to prefetch this file, don't try again
self._create_fetch_operation(key, site, filetype, obname)
def test_verify_wal(tmpdir):
b = BytesIO(WAL_HEADER_95 + b"XXX" * 100)
with pytest.raises(ValueError) as excinfo:
wal.verify_wal(wal_name="0" * 24, fileobj=b)
assert "found '11/9C000000'" in str(excinfo.value)
wal.verify_wal(wal_name="0000002F000000110000009C", fileobj=b)
tmp_file = tmpdir.join("xl").strpath
with open(tmp_file, "wb") as fp:
fp.write(b.getvalue())
wal.verify_wal(wal_name="0000002F000000110000009C", filepath=tmp_file)
with pytest.raises(ValueError) as excinfo:
wal.verify_wal(wal_name="0000002F000000110000009C", filepath=tmp_file + "x")
assert "FileNotFoundError" in str(excinfo.value)
def test_verify_wal(tmpdir):
b = BytesIO(WAL_HEADER_95 + b"XXX" * 100)
with pytest.raises(ValueError) as excinfo:
wal.verify_wal(wal_name="0" * 24, fileobj=b)
assert "found '11/9C000000'" in str(excinfo.value)
wal.verify_wal(wal_name="0000002F000000110000009C", fileobj=b)
tmp_file = tmpdir.join("xl").strpath
with open(tmp_file, "wb") as fp:
fp.write(b.getvalue())
wal.verify_wal(wal_name="0000002F000000110000009C", filepath=tmp_file)
with pytest.raises(ValueError) as excinfo:
wal.verify_wal(wal_name="0000002F000000110000009C", filepath=tmp_file + "x")
assert "FileNotFoundError" in str(excinfo.value)
def _create_prefetch_operations(self, site, filetype, filename):
if filetype not in {"timeline", "xlog"}:
return
prefetch_n = self.server.config["restore_prefetch"]
if prefetch_n <= 0:
return
xlog_dir = get_pg_wal_directory(self.server.config["backup_sites"][site])
names = []
if filetype == "timeline":
tli_num = int(filename.replace(".history", ""), 16)
for _ in range(prefetch_n):
tli_num += 1
prefetch_name = "{:08X}.history".format(tli_num)
if os.path.isfile(os.path.join(xlog_dir, prefetch_name)):
continue
names.append(prefetch_name)
elif filetype == "xlog":
xlog_num = int(filename, 16)
for _ in range(prefetch_n):
if xlog_num & 0xFF == 0xFF:
xlog_num += 0xFFFFFF00
xlog_num += 1
prefetch_name = "{:024X}".format(xlog_num)
xlog_path = os.path.join(xlog_dir, prefetch_name)
if os.path.isfile(xlog_path):
try:
wal.verify_wal(wal_name=prefetch_name, filepath=xlog_path)
continue
except ValueError as e:
self.server.log.debug("(Prefetch) File %s already exists but is invalid: %r", xlog_path, e)
names.append(prefetch_name)
for obname in names:
key = self._make_file_key(site, filetype, obname)
if key in self.server.prefetch_404:
continue # previously failed to prefetch this file, don't try again
self._create_fetch_operation(key, site, filetype, obname)
def handle_event(self, event, filetype):
# pylint: disable=redefined-variable-type
rsa_public_key = None
site = event.get("site")
if not site:
site = self.find_site_for_file(event["full_path"])
encryption_key_id = self.config["backup_sites"][site][
"encryption_key_id"]
if encryption_key_id:
rsa_public_key = self.config["backup_sites"][site][
"encryption_keys"][encryption_key_id]["public"]
compressed_blob = None
if event.get("compress_to_memory"):
output_obj = BytesIO()
compressed_filepath = None
else:
compressed_filepath = self.get_compressed_file_path(
site, filetype, event["full_path"])
output_obj = NamedTemporaryFile(
dir=os.path.dirname(compressed_filepath),
prefix=os.path.basename(compressed_filepath),
suffix=".tmp-compress")
input_obj = event.get("input_data")
if not input_obj:
input_obj = open(event["full_path"], "rb")
with output_obj, input_obj:
hash_algorithm = self.config["hash_algorithm"]
hasher = None
if filetype == "xlog":
wal.verify_wal(wal_name=os.path.basename(event["full_path"]),
fileobj=input_obj)
hasher = hashlib.new(hash_algorithm)
original_file_size, compressed_file_size = rohmufile.write_file(
data_callback=hasher.update if hasher else None,
input_obj=input_obj,
output_obj=output_obj,
compression_algorithm=self.config["compression"]["algorithm"],
compression_level=self.config["compression"]["level"],
rsa_public_key=rsa_public_key,
log_func=self.log.info,
)
if compressed_filepath:
os.link(output_obj.name, compressed_filepath)
else:
compressed_blob = output_obj.getvalue()
if event.get("delete_file_after_compression", True):
os.unlink(event["full_path"])
metadata = event.get("metadata", {})
metadata.update({
"pg-version":
self.config["backup_sites"][site].get("pg_version"),
"compression-algorithm":
self.config["compression"]["algorithm"],
"compression-level":
self.config["compression"]["level"],
"original-file-size":
original_file_size,
"host":
socket.gethostname(),
})
if hasher:
metadata["hash"] = hasher.hexdigest()
metadata["hash-algorithm"] = hash_algorithm
if encryption_key_id:
metadata.update({"encryption-key-id": encryption_key_id})
if compressed_filepath:
metadata_path = compressed_filepath + ".metadata"
write_json_file(metadata_path, metadata)
self.set_state_defaults_for_site(site)
self.state[site][filetype]["original_data"] += original_file_size
self.state[site][filetype]["compressed_data"] += compressed_file_size
self.state[site][filetype]["count"] += 1
if original_file_size:
size_ratio = compressed_file_size / original_file_size
self.metrics.gauge("pghoard.compressed_size_ratio",
size_ratio,
tags={
"algorithm":
self.config["compression"]["algorithm"],
"site":
site,
"type":
filetype,
})
transfer_object = {
"callback_queue": event.get("callback_queue"),
"file_size": compressed_file_size,
"filetype": filetype,
"metadata": metadata,
"opaque": event.get("opaque"),
"site": site,
"type": "UPLOAD",
}
if compressed_filepath:
transfer_object["local_path"] = compressed_filepath
else:
transfer_object["blob"] = compressed_blob
transfer_object["local_path"] = event["full_path"]
self.transfer_queue.put(transfer_object)
return True
def handle_event(self, event, filetype):
# pylint: disable=redefined-variable-type
rsa_public_key = None
site = event.get("site")
if not site:
site = self.find_site_for_file(event["full_path"])
encryption_key_id = self.config["backup_sites"][site]["encryption_key_id"]
if encryption_key_id:
rsa_public_key = self.config["backup_sites"][site]["encryption_keys"][encryption_key_id]["public"]
compressed_blob = None
if event.get("compress_to_memory"):
output_obj = BytesIO()
compressed_filepath = None
else:
compressed_filepath = self.get_compressed_file_path(site, filetype, event["full_path"])
output_obj = NamedTemporaryFile(dir=os.path.dirname(compressed_filepath),
prefix=os.path.basename(compressed_filepath), suffix=".tmp-compress")
input_obj = event.get("input_data")
if not input_obj:
input_obj = open(event["full_path"], "rb")
with output_obj, input_obj:
hash_algorithm = self.config["hash_algorithm"]
hasher = None
if filetype == "xlog":
wal.verify_wal(wal_name=os.path.basename(event["full_path"]), fileobj=input_obj)
hasher = hashlib.new(hash_algorithm)
original_file_size, compressed_file_size = rohmufile.write_file(
data_callback=hasher.update if hasher else None,
input_obj=input_obj,
output_obj=output_obj,
compression_algorithm=self.config["compression"]["algorithm"],
compression_level=self.config["compression"]["level"],
rsa_public_key=rsa_public_key,
log_func=self.log.info,
)
if compressed_filepath:
os.link(output_obj.name, compressed_filepath)
else:
compressed_blob = output_obj.getvalue()
if event.get("delete_file_after_compression", True):
os.unlink(event["full_path"])
metadata = event.get("metadata", {})
metadata.update({
"pg-version": self.config["backup_sites"][site].get("pg_version"),
"compression-algorithm": self.config["compression"]["algorithm"],
"compression-level": self.config["compression"]["level"],
"original-file-size": original_file_size,
"host": socket.gethostname(),
})
if hasher:
metadata["hash"] = hasher.hexdigest()
metadata["hash-algorithm"] = hash_algorithm
if encryption_key_id:
metadata.update({"encryption-key-id": encryption_key_id})
if compressed_filepath:
metadata_path = compressed_filepath + ".metadata"
write_json_file(metadata_path, metadata)
self.set_state_defaults_for_site(site)
self.state[site][filetype]["original_data"] += original_file_size
self.state[site][filetype]["compressed_data"] += compressed_file_size
self.state[site][filetype]["count"] += 1
if original_file_size:
size_ratio = compressed_file_size / original_file_size
self.metrics.gauge(
"pghoard.compressed_size_ratio", size_ratio,
tags={
"algorithm": self.config["compression"]["algorithm"],
"site": site,
"type": filetype,
})
transfer_object = {
"callback_queue": event.get("callback_queue"),
"file_size": compressed_file_size,
"filetype": filetype,
"metadata": metadata,
"opaque": event.get("opaque"),
"site": site,
"type": "UPLOAD",
}
if compressed_filepath:
transfer_object["local_path"] = compressed_filepath
else:
transfer_object["blob"] = compressed_blob
transfer_object["local_path"] = event["full_path"]
self.transfer_queue.put(transfer_object)
return True
def test_verify_wal_starts_moves_fp_back():
with TemporaryFile("w+b") as tmp_file:
tmp_file.write(WAL_HEADER_95 + b"XXX" * 100)
tmp_file.seek(10)
wal.verify_wal(wal_name="0000002F000000110000009C", fileobj=tmp_file)
assert tmp_file.tell() == 10
def test_verify_wal_starts_at_bof():
with TemporaryFile("w+b") as tmp_file:
tmp_file.write(WAL_HEADER_95 + b"XXX" * 100)
tmp_file.seek(10)
wal.verify_wal(wal_name="0000002F000000110000009C", fileobj=tmp_file)