def test_decryptorfile_for_tarfile(tmpdir):
testdata = b"file contents"
data_tmp_name = tmpdir.join("plain.data").strpath
with open(data_tmp_name, mode="wb") as data_tmp:
data_tmp.write(testdata)
tar_data = io.BytesIO()
with tarfile.open(name="foo", fileobj=tar_data, mode="w") as tar:
tar.add(data_tmp_name, arcname="archived_content")
plaintext = tar_data.getvalue()
encryptor = Encryptor(CONSTANT_TEST_RSA_PUBLIC_KEY)
ciphertext = encryptor.update(plaintext) + encryptor.finalize()
enc_tar_name = tmpdir.join("enc.tar.data").strpath
with open(enc_tar_name, "w+b") as enc_tar:
enc_tar.write(ciphertext)
enc_tar.seek(0)
dfile = DecryptorFile(enc_tar, CONSTANT_TEST_RSA_PRIVATE_KEY)
with tarfile.open(fileobj=dfile, mode="r") as tar:
info = tar.getmember("archived_content")
assert info.isfile() is True
assert info.size == len(testdata)
content_file = tar.extractfile("archived_content")
content = content_file.read() # pylint: disable=no-member
content_file.close() # pylint: disable=no-member
assert testdata == content
decout = tmpdir.join("dec_out_dir").strpath
os.makedirs(decout)
tar.extract("archived_content", decout)
extracted_path = os.path.join(decout, "archived_content")
with open(extracted_path, "rb") as ext_fp:
assert testdata == ext_fp.read()
def test_get_encrypted_archived_file(self, pghoard):
xlog_seg = "000000090000000000000010"
content = wal_header_for_file(xlog_seg)
compressor = pghoard.Compressor()
compressed_content = compressor.compress(content) + (compressor.flush() or b"")
encryptor = Encryptor(CONSTANT_TEST_RSA_PUBLIC_KEY)
encrypted_content = encryptor.update(compressed_content) + encryptor.finalize()
pgdata = os.path.dirname(pghoard.config["backup_sites"][pghoard.test_site]["pg_xlog_directory"])
archive_path = os.path.join(pghoard.test_site, "xlog", xlog_seg)
metadata = {
"compression-algorithm": pghoard.config["compression"]["algorithm"],
"original-file-size": len(content),
"encryption-key-id": "testkey",
}
store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
store.store_file_from_memory(archive_path, encrypted_content, metadata=metadata)
pghoard.webserver.config["backup_sites"][pghoard.test_site]["encryption_keys"] = {
"testkey": {
"public": CONSTANT_TEST_RSA_PUBLIC_KEY,
"private": CONSTANT_TEST_RSA_PRIVATE_KEY,
},
}
restore_target = os.path.join(pgdata, "pg_xlog", xlog_seg)
restore_command(site=pghoard.test_site, xlog=xlog_seg, output=restore_target,
host="127.0.0.1", port=pghoard.config["http_port"])
assert os.path.exists(restore_target)
with open(restore_target, "rb") as fp:
restored_data = fp.read()
assert content == restored_data
def test_decryptorfile(self):
# create a plaintext blob bigger than IO_BLOCK_SIZE
plaintext1 = b"rvdmfki6iudmx8bb25tx1sozex3f4u0nm7uba4eibscgda0ckledcydz089qw1p1"
repeat = int(1.5 * IO_BLOCK_SIZE / len(plaintext1))
plaintext = repeat * plaintext1
encryptor = Encryptor(CONSTANT_TEST_RSA_PUBLIC_KEY)
ciphertext = encryptor.update(plaintext) + encryptor.finalize()
plain_fp = tempfile.TemporaryFile(prefix="test-pghoard.", mode="r+b")
plain_fp.write(ciphertext)
plain_fp.seek(0)
fp = DecryptorFile(plain_fp, CONSTANT_TEST_RSA_PRIVATE_KEY) # pylint: disable=redefined-variable-type
assert fp.fileno() == plain_fp.fileno()
assert fp.readable() is True
assert fp.writable() is False
fp.flush()
result = fp.read()
assert plaintext == result
assert fp.seekable() is True
with pytest.raises(ValueError):
fp.seek(-1)
fp.seek(0, os.SEEK_SET)
with pytest.raises(io.UnsupportedOperation):
fp.seek(1, os.SEEK_CUR)
with pytest.raises(io.UnsupportedOperation):
fp.seek(1, os.SEEK_END)
with pytest.raises(ValueError):
fp.seek(1, 0xff)
assert fp.seek(0, os.SEEK_END) == len(plaintext)
assert fp.seek(0, os.SEEK_CUR) == len(plaintext)
fp.seek(0)
result = fp.read()
assert plaintext == result
assert fp.read(1234) == b""
assert fp.read() == b""
fp.seek(0)
result = fp.read(8192)
assert result == plaintext[:8192]
result = fp.read(8192)
assert result == plaintext[8192:8192 * 2]
result = fp.read(IO_BLOCK_SIZE * 2)
assert plaintext[8192 * 2:] == result
assert fp.seek(IO_BLOCK_SIZE // 2) == IO_BLOCK_SIZE // 2
result = fp.read()
assert len(result) == len(plaintext) - IO_BLOCK_SIZE // 2
assert plaintext[IO_BLOCK_SIZE // 2:] == result
fp.seek(2)
result = fp.read(1)
assert plaintext[2:3] == result
assert fp.tell() == 3
with pytest.raises(io.UnsupportedOperation):
fp.truncate()
# close the file (this can be safely called multiple times), other ops should fail after that
fp.close()
fp.close()
with pytest.raises(ValueError):
fp.truncate()
def test_decryptorfile_for_tarfile(self):
testdata = b"file contents"
data_tmp = tempfile.NamedTemporaryFile(prefix="test-pghoard.",
mode="r+b")
data_tmp.write(testdata)
data_tmp.flush()
tmp = tempfile.TemporaryFile(prefix="test-pghoard.", mode="r+b")
tar = tarfile.open(name="foo", fileobj=tmp, mode="w")
tar.add(data_tmp.name, arcname="archived_content")
tar.close()
tmp.seek(0)
plaintext = tmp.read()
tmp.seek(0)
tmp.truncate()
encryptor = Encryptor(CONSTANT_TEST_RSA_PUBLIC_KEY)
ciphertext = encryptor.update(plaintext) + encryptor.finalize()
tmp.write(ciphertext)
tmp.seek(0)
tmp = DecryptorFile(tmp, CONSTANT_TEST_RSA_PRIVATE_KEY) # pylint: disable=redefined-variable-type
tar = tarfile.open(fileobj=tmp, mode="r")
info = tar.getmember("archived_content")
assert info.isfile() is True
assert info.size == len(testdata)
content_file = tar.extractfile("archived_content")
tar.extract("archived_content", "/tmp/testout")
content = content_file.read() # pylint: disable=no-member
content_file.close() # pylint: disable=no-member
assert testdata == content
tar.close()
tmp.close()
data_tmp.close()
def test_encryptor_decryptor():
plaintext = b"test"
for op in (None, "json"):
if op == "json":
public_key = json.loads(json.dumps(CONSTANT_TEST_RSA_PUBLIC_KEY))
private_key = json.loads(json.dumps(CONSTANT_TEST_RSA_PRIVATE_KEY))
else:
public_key = CONSTANT_TEST_RSA_PUBLIC_KEY
private_key = CONSTANT_TEST_RSA_PRIVATE_KEY
encryptor = Encryptor(public_key)
decryptor = Decryptor(private_key)
encrypted = encryptor.update(plaintext) + encryptor.finalize()
assert plaintext not in encrypted
offset = 0
while decryptor.expected_header_bytes() > 0:
chunk = encrypted[offset:offset +
decryptor.expected_header_bytes()]
decryptor.process_header(chunk)
offset += len(chunk)
decrypted_size = len(
encrypted) - decryptor.header_size() - decryptor.footer_size()
decrypted = decryptor.process_data(
encrypted[decryptor.header_size():decryptor.header_size() +
decrypted_size])
decrypted += decryptor.finalize(encrypted[-decryptor.footer_size():])
assert plaintext == decrypted
def test_decryptorfile(tmpdir):
# create a plaintext blob bigger than IO_BLOCK_SIZE
plaintext1 = b"rvdmfki6iudmx8bb25tx1sozex3f4u0nm7uba4eibscgda0ckledcydz089qw1p1"
repeat = int(1.5 * IO_BLOCK_SIZE / len(plaintext1))
plaintext = repeat * plaintext1
encryptor = Encryptor(CONSTANT_TEST_RSA_PUBLIC_KEY)
ciphertext = encryptor.update(plaintext) + encryptor.finalize()
plain_fp = open(tmpdir.join("plain").strpath, mode="w+b")
plain_fp.write(ciphertext)
plain_fp.seek(0)
fp = DecryptorFile(plain_fp, CONSTANT_TEST_RSA_PRIVATE_KEY) # pylint: disable=redefined-variable-type
assert fp.fileno() == plain_fp.fileno()
assert fp.readable() is True
assert fp.writable() is False
fp.flush()
result = fp.read()
assert plaintext == result
assert fp.seekable() is True
with pytest.raises(ValueError):
fp.seek(-1)
fp.seek(0, os.SEEK_SET)
with pytest.raises(io.UnsupportedOperation):
fp.seek(1, os.SEEK_CUR)
with pytest.raises(io.UnsupportedOperation):
fp.seek(1, os.SEEK_END)
with pytest.raises(ValueError):
fp.seek(1, 0xff)
assert fp.seek(0, os.SEEK_END) == len(plaintext)
assert fp.seek(0, os.SEEK_CUR) == len(plaintext)
fp.seek(0)
result = fp.read()
assert plaintext == result
assert fp.read(1234) == b""
assert fp.read() == b""
fp.seek(0)
result = fp.read(8192)
assert result == plaintext[:8192]
result = fp.read(8192)
assert result == plaintext[8192:8192 * 2]
result = fp.read(IO_BLOCK_SIZE * 2)
assert plaintext[8192 * 2:] == result
assert fp.seek(IO_BLOCK_SIZE // 2) == IO_BLOCK_SIZE // 2
result = fp.read()
assert len(result) == len(plaintext) - IO_BLOCK_SIZE // 2
assert plaintext[IO_BLOCK_SIZE // 2:] == result
fp.seek(2)
result = fp.read(1)
assert plaintext[2:3] == result
assert fp.tell() == 3
with pytest.raises(io.UnsupportedOperation):
fp.truncate()
# close the file (this can be safely called multiple times), other ops should fail after that
fp.close()
fp.close()
with pytest.raises(ValueError):
fp.truncate()
def test_decryptorfile_for_tarfile(tmpdir):
testdata = b"file contents"
data_tmp_name = tmpdir.join("plain.data").strpath
with open(data_tmp_name, mode="wb") as data_tmp:
data_tmp.write(testdata)
tar_data = io.BytesIO()
with tarfile.open(name="foo", fileobj=tar_data, mode="w") as tar:
tar.add(data_tmp_name, arcname="archived_content")
plaintext = tar_data.getvalue()
encryptor = Encryptor(CONSTANT_TEST_RSA_PUBLIC_KEY)
ciphertext = encryptor.update(plaintext) + encryptor.finalize()
enc_tar_name = tmpdir.join("enc.tar.data").strpath
with open(enc_tar_name, "w+b") as enc_tar:
enc_tar.write(ciphertext)
enc_tar.seek(0)
dfile = DecryptorFile(enc_tar, CONSTANT_TEST_RSA_PRIVATE_KEY)
with tarfile.open(fileobj=dfile, mode="r") as tar:
info = tar.getmember("archived_content")
assert info.isfile() is True
assert info.size == len(testdata)
content_file = tar.extractfile("archived_content")
content = content_file.read() # pylint: disable=no-member
content_file.close() # pylint: disable=no-member
assert testdata == content
decout = tmpdir.join("dec_out_dir").strpath
os.makedirs(decout)
tar.extract("archived_content", decout)
extracted_path = os.path.join(decout, "archived_content")
with open(extracted_path, "rb") as ext_fp:
assert testdata == ext_fp.read()
def test_get_encrypted_archived_file(self, pghoard):
wal_seg = "000000090000000000000010"
content = wal_header_for_file(wal_seg)
compressor = pghoard.Compressor()
compressed_content = compressor.compress(content) + (compressor.flush() or b"")
encryptor = Encryptor(CONSTANT_TEST_RSA_PUBLIC_KEY)
encrypted_content = encryptor.update(compressed_content) + encryptor.finalize()
wal_dir = get_pg_wal_directory(pghoard.config["backup_sites"][pghoard.test_site])
archive_path = os.path.join(pghoard.test_site, "xlog", wal_seg)
metadata = {
"compression-algorithm": pghoard.config["compression"]["algorithm"],
"original-file-size": len(content),
"encryption-key-id": "testkey",
}
store = pghoard.transfer_agents[0].get_object_storage(pghoard.test_site)
store.store_file_from_memory(archive_path, encrypted_content, metadata=metadata)
pghoard.webserver.config["backup_sites"][pghoard.test_site]["encryption_keys"] = {
"testkey": {
"public": CONSTANT_TEST_RSA_PUBLIC_KEY,
"private": CONSTANT_TEST_RSA_PRIVATE_KEY,
},
}
restore_target = os.path.join(wal_dir, wal_seg)
restore_command(site=pghoard.test_site, xlog=wal_seg, output=restore_target,
host="127.0.0.1", port=pghoard.config["http_port"])
assert os.path.exists(restore_target)
with open(restore_target, "rb") as fp:
restored_data = fp.read()
assert content == restored_data
def test_decryptorfile_for_tarfile(self):
testdata = b"file contents"
data_tmp = tempfile.NamedTemporaryFile(prefix="test-pghoard.", mode="r+b")
data_tmp.write(testdata)
data_tmp.flush()
tmp = tempfile.TemporaryFile(prefix="test-pghoard.", mode="r+b")
tar = tarfile.open(name="foo", fileobj=tmp, mode="w")
tar.add(data_tmp.name, arcname="archived_content")
tar.close()
tmp.seek(0)
plaintext = tmp.read()
tmp.seek(0)
tmp.truncate()
encryptor = Encryptor(CONSTANT_TEST_RSA_PUBLIC_KEY)
ciphertext = encryptor.update(plaintext) + encryptor.finalize()
tmp.write(ciphertext)
tmp.seek(0)
tmp = DecryptorFile(tmp, CONSTANT_TEST_RSA_PRIVATE_KEY) # pylint: disable=redefined-variable-type
tar = tarfile.open(fileobj=tmp, mode="r")
info = tar.getmember("archived_content")
assert info.isfile() is True
assert info.size == len(testdata)
content_file = tar.extractfile("archived_content")
tar.extract("archived_content", "/tmp/testout")
content = content_file.read() # pylint: disable=no-member
content_file.close() # pylint: disable=no-member
assert testdata == content
tar.close()
tmp.close()
data_tmp.close()
def test_decryptorfile(self):
plaintext = b"test"
encryptor = Encryptor(CONSTANT_TEST_RSA_PUBLIC_KEY)
ciphertext = encryptor.update(plaintext) + encryptor.finalize()
fp = tempfile.TemporaryFile(prefix="test-pghoard.", mode="r+b")
fp.write(ciphertext)
fp.seek(0)
fp = DecryptorFile(fp, CONSTANT_TEST_RSA_PRIVATE_KEY) # pylint: disable=redefined-variable-type
result = fp.read()
assert plaintext == result
fp.seek(0)
result = fp.read()
assert plaintext == result
fp.seek(2)
result = fp.read(1)
assert plaintext[2:3] == result
def test_encryptor_decryptor(self):
plaintext = b"test"
encryptor = Encryptor(CONSTANT_TEST_RSA_PUBLIC_KEY)
ciphertext = encryptor.update(plaintext) + encryptor.finalize()
assert plaintext not in ciphertext
decryptor = Decryptor(CONSTANT_TEST_RSA_PRIVATE_KEY)
result = decryptor.update(ciphertext) + decryptor.finalize()
assert plaintext == result
public_key = json.loads(json.dumps(CONSTANT_TEST_RSA_PUBLIC_KEY))
private_key = json.loads(json.dumps(CONSTANT_TEST_RSA_PRIVATE_KEY))
encryptor = Encryptor(public_key)
decryptor = Decryptor(private_key)
assert plaintext == decryptor.update(
encryptor.update(plaintext) +
encryptor.finalize()) + decryptor.finalize()
def test_encryptor_decryptor():
plaintext = b"test"
for op in (None, "json"):
if op == "json":
public_key = json.loads(json.dumps(CONSTANT_TEST_RSA_PUBLIC_KEY))
private_key = json.loads(json.dumps(CONSTANT_TEST_RSA_PRIVATE_KEY))
else:
public_key = CONSTANT_TEST_RSA_PUBLIC_KEY
private_key = CONSTANT_TEST_RSA_PRIVATE_KEY
encryptor = Encryptor(public_key)
decryptor = Decryptor(private_key)
encrypted = encryptor.update(plaintext) + encryptor.finalize()
assert plaintext not in encrypted
offset = 0
while decryptor.expected_header_bytes() > 0:
chunk = encrypted[offset:offset + decryptor.expected_header_bytes()]
decryptor.process_header(chunk)
offset += len(chunk)
decrypted_size = len(encrypted) - decryptor.header_size() - decryptor.footer_size()
decrypted = decryptor.process_data(encrypted[decryptor.header_size():decryptor.header_size() + decrypted_size])
decrypted += decryptor.finalize(encrypted[-decryptor.footer_size():])
assert plaintext == decrypted
def test_encryptor_decryptor():
plaintext = b"test"
encryptor = Encryptor(CONSTANT_TEST_RSA_PUBLIC_KEY)
ciphertext = encryptor.update(plaintext) + encryptor.finalize()
assert plaintext not in ciphertext
decryptor = Decryptor(CONSTANT_TEST_RSA_PRIVATE_KEY)
result = decryptor.update(ciphertext) + decryptor.finalize()
assert plaintext == result
public_key = json.loads(json.dumps(CONSTANT_TEST_RSA_PUBLIC_KEY))
private_key = json.loads(json.dumps(CONSTANT_TEST_RSA_PRIVATE_KEY))
encryptor = Encryptor(public_key)
decryptor = Decryptor(private_key)
assert plaintext == decryptor.update(encryptor.update(plaintext) + encryptor.finalize()) + decryptor.finalize()
