def compute_storage_size(cls,
block_size,
heap_height,
blocks_per_bucket=1,
heap_base=2,
ignore_header=False,
**kwds):
assert (block_size > 0) and (block_size == int(block_size))
assert heap_height >= 0
assert blocks_per_bucket >= 1
assert heap_base >= 2
assert 'block_count' not in kwds
vheap = SizedVirtualHeap(heap_base,
heap_height,
blocks_per_bucket=blocks_per_bucket)
if ignore_header:
return EncryptedBlockStorage.compute_storage_size(
vheap.blocks_per_bucket * block_size,
vheap.bucket_count(),
ignore_header=True,
**kwds)
else:
return cls._header_offset + \
EncryptedBlockStorage.compute_storage_size(
vheap.blocks_per_bucket * block_size,
vheap.bucket_count(),
ignore_header=False,
**kwds)
def compute_storage_size(cls,
block_size,
heap_height,
blocks_per_bucket=1,
heap_base=2,
ignore_header=False,
**kwds):
assert (block_size > 0) and (block_size == int(block_size))
assert heap_height >= 0
assert blocks_per_bucket >= 1
assert heap_base >= 2
assert 'block_count' not in kwds
vheap = SizedVirtualHeap(
heap_base,
heap_height,
blocks_per_bucket=blocks_per_bucket)
if ignore_header:
return EncryptedBlockStorage.compute_storage_size(
vheap.blocks_per_bucket * block_size,
vheap.bucket_count(),
ignore_header=True,
**kwds)
else:
return cls._header_offset + \
EncryptedBlockStorage.compute_storage_size(
vheap.blocks_per_bucket * block_size,
vheap.bucket_count(),
ignore_header=False,
**kwds)
def test_setup(self):
fname = ".".join(self.id().split(".")[1:])
fname += ".bin"
fname = os.path.join(thisdir, fname)
if os.path.exists(fname):
os.remove(fname) # pragma: no cover
bsize = 10
bcount = 11
fsetup = EncryptedBlockStorage.setup(
fname,
bsize,
bcount,
key=self._test_key,
key_size=self._test_key_size,
aes_mode=self._aes_mode,
storage_type=self._type_name)
fsetup.close()
self.assertEqual(type(fsetup.raw_storage),
BlockStorageTypeFactory(self._type_name))
with open(fname, 'rb') as f:
flen = len(f.read())
self.assertEqual(
flen,
EncryptedBlockStorage.compute_storage_size(
bsize,
bcount,
aes_mode=self._aes_mode,
storage_type=self._type_name))
self.assertEqual(
flen >
EncryptedBlockStorage.compute_storage_size(
bsize,
bcount,
aes_mode=self._aes_mode,
storage_type=self._type_name,
ignore_header=True),
True)
with EncryptedBlockStorage(fname,
key=fsetup.key,
storage_type=self._type_name) as f:
self.assertEqual(f.header_data, bytes())
self.assertEqual(fsetup.header_data, bytes())
self.assertEqual(f.key, fsetup.key)
self.assertEqual(f.block_size, bsize)
self.assertEqual(fsetup.block_size, bsize)
self.assertEqual(f.block_count, bcount)
self.assertEqual(fsetup.block_count, bcount)
self.assertEqual(f.storage_name, fname)
self.assertEqual(fsetup.storage_name, fname)
# tamper with the plaintext index
with open(fname, 'r+b') as f:
f.seek(0)
f.write(struct.pack("!L",0))
with self.assertRaises(ValueError):
with EncryptedBlockStorage(fname,
key=fsetup.key,
storage_type=self._type_name) as f:
pass # pragma: no cover
os.remove(fname)
def main():
print("Storage Name: %s" % (storage_name))
print("Block Count: %s" % (block_count))
print("Block Size: %s" % (MemorySize(block_size)))
print("Total Memory: %s"
% (MemorySize(block_size*block_count)))
print("Actual Storage Required: %s"
% (MemorySize(
EncryptedBlockStorage.compute_storage_size(
block_size,
block_count,
storage_type='sftp'))))
print("")
# Start an SSH client using paramiko
print("Starting SSH Client")
with paramiko.SSHClient() as ssh:
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.load_system_host_keys()
ssh.connect(ssh_host,
username=ssh_username,
password=ssh_password)
print("Setting Up Encrypted Block Storage")
setup_start = time.time()
with EncryptedBlockStorage.setup(storage_name,
block_size,
block_count,
storage_type='sftp',
sshclient=ssh,
ignore_existing=True) as f:
print("Total Setup Time: %2.f s"
% (time.time()-setup_start))
print("Total Data Transmission: %s"
% (MemorySize(f.bytes_sent + f.bytes_received)))
print("")
# We close the device and reopen it after
# setup to reset the bytes sent and bytes
# received stats.
with EncryptedBlockStorage(storage_name,
key=f.key,
storage_type='sftp',
sshclient=ssh) as f:
test_count = 1000
start_time = time.time()
for t in tqdm.tqdm(list(range(test_count)),
desc="Running I/O Performance Test"):
f.read_block(random.randint(0,f.block_count-1))
stop_time = time.time()
print("Access Block Avg. Data Transmitted: %s (%.3fx)"
% (MemorySize((f.bytes_sent + f.bytes_received)/float(test_count)),
(f.bytes_sent + f.bytes_received)/float(test_count)/float(block_size)))
print("Access Block Avg. Latency: %.2f ms"
% ((stop_time-start_time)/float(test_count)*1000))
print("")
def main():
print("Storage Name: %s" % (storage_name))
print("Block Count: %s" % (block_count))
print("Block Size: %s" % (MemorySize(block_size)))
print("Total Memory: %s"
% (MemorySize(block_size*block_count)))
print("Actual Storage Required: %s"
% (MemorySize(
EncryptedBlockStorage.compute_storage_size(
block_size,
block_count,
storage_type='ram'))))
print("")
print("Setting Up Encrypted Block Storage")
setup_start = time.time()
with EncryptedBlockStorage.setup(storage_name, # RAM storage ignores this argument
block_size,
block_count,
storage_type='ram',
ignore_existing=True) as f:
print("Total Setup Time: %2.f s"
% (time.time()-setup_start))
print("Total Data Transmission: %s"
% (MemorySize(f.bytes_sent + f.bytes_received)))
print("")
# This must be done after closing the file to ensure the lock flag
# is set to False in the saved data. The tofile method only exists
# on BlockStorageRAM
f.raw_storage.tofile(storage_name)
# We close the device and reopen it after
# setup to reset the bytes sent and bytes
# received stats.
with EncryptedBlockStorage(BlockStorageRAM.fromfile(storage_name),
key=f.key) as f:
test_count = 1000
start_time = time.time()
for t in tqdm.tqdm(list(range(test_count)),
desc="Running I/O Performance Test"):
f.read_block(random.randint(0,f.block_count-1))
stop_time = time.time()
print("Access Block Avg. Data Transmitted: %s (%.3fx)"
% (MemorySize((f.bytes_sent + f.bytes_received)/float(test_count)),
(f.bytes_sent + f.bytes_received)/float(test_count)/float(block_size)))
print("Access Block Avg. Latency: %.2f ms"
% ((stop_time-start_time)/float(test_count)*1000))
print("")
# cleanup because this is a test example
os.remove(storage_name)
def main():
print("Storage Name: %s" % (storage_name))
print("Block Count: %s" % (block_count))
print("Block Size: %s" % (MemorySize(block_size)))
print("Total Memory: %s" % (MemorySize(block_size * block_count)))
print("Actual Storage Required: %s" % (MemorySize(
EncryptedBlockStorage.compute_storage_size(
block_size, block_count, storage_type='sftp'))))
print("")
# Start an SSH client using paramiko
print("Starting SSH Client")
with paramiko.SSHClient() as ssh:
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.load_system_host_keys()
ssh.connect(ssh_host, username=ssh_username, password=ssh_password)
print("Setting Up Encrypted Block Storage")
setup_start = time.time()
with EncryptedBlockStorage.setup(storage_name,
block_size,
block_count,
storage_type='sftp',
sshclient=ssh,
ignore_existing=True) as f:
print("Total Setup Time: %2.f s" % (time.time() - setup_start))
print("Total Data Transmission: %s" %
(MemorySize(f.bytes_sent + f.bytes_received)))
print("")
# We close the device and reopen it after
# setup to reset the bytes sent and bytes
# received stats.
with EncryptedBlockStorage(storage_name,
key=f.key,
storage_type='sftp',
sshclient=ssh) as f:
test_count = 1000
start_time = time.time()
for t in tqdm.tqdm(list(range(test_count)),
desc="Running I/O Performance Test"):
f.read_block(random.randint(0, f.block_count - 1))
stop_time = time.time()
print("Access Block Avg. Data Transmitted: %s (%.3fx)" %
(MemorySize(
(f.bytes_sent + f.bytes_received) / float(test_count)),
(f.bytes_sent + f.bytes_received) / float(test_count) /
float(block_size)))
print("Access Block Avg. Latency: %.2f ms" %
((stop_time - start_time) / float(test_count) * 1000))
print("")
def main():
print("Storage Name: %s" % (storage_name))
print("Block Count: %s" % (block_count))
print("Block Size: %s" % (MemorySize(block_size)))
print("Total Memory: %s"
% (MemorySize(block_size*block_count)))
print("Actual Storage Required: %s"
% (MemorySize(
EncryptedBlockStorage.compute_storage_size(
block_size,
block_count,
storage_type='mmap'))))
print("")
print("Setting Up Encrypted Block Storage")
setup_start = time.time()
with EncryptedBlockStorage.setup(storage_name,
block_size,
block_count,
storage_type='mmap',
ignore_existing=True) as f:
print("Total Setup Time: %2.f s"
% (time.time()-setup_start))
print("Total Data Transmission: %s"
% (MemorySize(f.bytes_sent + f.bytes_received)))
print("")
# We close the device and reopen it after
# setup to reset the bytes sent and bytes
# received stats.
with EncryptedBlockStorage(storage_name,
key=f.key,
storage_type='mmap') as f:
test_count = 1000
start_time = time.time()
for t in tqdm.tqdm(list(range(test_count)),
desc="Running I/O Performance Test"):
f.read_block(random.randint(0,f.block_count-1))
stop_time = time.time()
print("Access Block Avg. Data Transmitted: %s (%.3fx)"
% (MemorySize((f.bytes_sent + f.bytes_received)/float(test_count)),
(f.bytes_sent + f.bytes_received)/float(test_count)/float(block_size)))
print("Access Block Avg. Latency: %.2f ms"
% ((stop_time-start_time)/float(test_count)*1000))
print("")
# cleanup because this is a test example
os.remove(storage_name)
def setUpClass(cls):
assert cls._type_name is not None
assert cls._aes_mode is not None
assert not ((cls._test_key is not None) and \
(cls._test_key_size is not None))
fd, cls._dummy_name = tempfile.mkstemp()
os.close(fd)
try:
os.remove(cls._dummy_name)
except OSError: # pragma: no cover
pass # pragma: no cover
cls._block_size = 25
cls._block_count = 5
cls._testfname = cls.__name__ + "_testfile.bin"
cls._blocks = []
f = EncryptedBlockStorage.setup(
cls._testfname,
cls._block_size,
cls._block_count,
key_size=cls._test_key_size,
key=cls._test_key,
storage_type=cls._type_name,
aes_mode=cls._aes_mode,
initialize=lambda i: bytes(bytearray([i])*cls._block_size),
ignore_existing=True)
f.close()
cls._key = f.key
for i in range(cls._block_count):
data = bytearray([i])*cls._block_size
cls._blocks.append(data)
def main():
print("Storage Name: %s" % (storage_name))
print("Block Count: %s" % (block_count))
print("Block Size: %s" % (MemorySize(block_size)))
print("Total Memory: %s" % (MemorySize(block_size * block_count)))
print("Actual Storage Required: %s" % (MemorySize(
EncryptedBlockStorage.compute_storage_size(
block_size, block_count, storage_type='s3'))))
print("")
print("Setting Up Encrypted Block Storage")
setup_start = time.time()
with EncryptedBlockStorage.setup(storage_name,
block_size,
block_count,
storage_type='s3',
bucket_name=bucket_name,
ignore_existing=True) as f:
print("Total Setup Time: %2.f s" % (time.time() - setup_start))
print("Total Data Transmission: %s" %
(MemorySize(f.bytes_sent + f.bytes_received)))
print("")
# We close the device and reopen it after
# setup to reset the bytes sent and bytes
# received stats.
with EncryptedBlockStorage(storage_name,
key=f.key,
storage_type='s3',
bucket_name=bucket_name) as f:
test_count = 1000
start_time = time.time()
for t in tqdm.tqdm(list(range(test_count)),
desc="Running I/O Performance Test"):
f.read_block(random.randint(0, f.block_count - 1))
stop_time = time.time()
print("Access Block Avg. Data Transmitted: %s (%.3fx)" %
(MemorySize(
(f.bytes_sent + f.bytes_received) / float(test_count)),
(f.bytes_sent + f.bytes_received) / float(test_count) /
float(block_size)))
print("Access Block Avg. Latency: %.2f ms" %
((stop_time - start_time) / float(test_count) * 1000))
print("")
def __init__(self, storage, **kwds):
if isinstance(storage, EncryptedBlockStorageInterface):
if len(kwds):
raise ValueError("Keywords not used when initializing "
"with a storage device: %s" % (str(kwds)))
else:
storage = EncryptedBlockStorage(storage, **kwds)
super(EncryptedHeapStorage, self).__init__(storage)
def test_init_exists(self):
self.assertEqual(os.path.exists(self._testfname), True)
with EncryptedBlockStorage(self._testfname,
key=self._key,
storage_type=self._storage_type) as f:
databefore = f.read_blocks(list(range(f.block_count)))
with self.assertRaises(ValueError):
with EncryptedBlockStorage(self._testfname,
key=self._key,
storage_type=self._storage_type) as fb:
with TopCachedEncryptedHeapStorage(
EncryptedHeapStorage(fb, key=self._key),
**self._init_kwds) as f:
pass # pragma: no cover
with TopCachedEncryptedHeapStorage(
EncryptedHeapStorage(
self._testfname,
key=self._key,
storage_type=self._storage_type),
**self._init_kwds) as f:
self.assertEqual(f.key, self._key)
self.assertEqual(f.bucket_size,
self._block_size * \
self._blocks_per_bucket)
self.assertEqual(f.bucket_count,
self._bucket_count)
self.assertEqual(f.storage_name, self._testfname)
self.assertEqual(f.header_data, bytes())
self.assertEqual(os.path.exists(self._testfname), True)
with TopCachedEncryptedHeapStorage(
EncryptedHeapStorage(
self._testfname,
key=self._key,
storage_type=self._storage_type),
**self._init_kwds) as f:
dataafter = f.bucket_storage.read_blocks(
list(range(f.bucket_storage.block_count)))
self.assertEqual(databefore, dataafter)
def setup(cls,
storage_name,
block_size,
heap_height,
blocks_per_bucket=1,
heap_base=2,
**kwds):
if 'block_count' in kwds:
raise ValueError("'block_count' keyword is not accepted")
if heap_height < 0:
raise ValueError(
"heap height must be 0 or greater. Invalid value: %s"
% (heap_height))
if blocks_per_bucket < 1:
raise ValueError(
"blocks_per_bucket must be 1 or greater. "
"Invalid value: %s" % (blocks_per_bucket))
if heap_base < 2:
raise ValueError(
"heap base must be 2 or greater. Invalid value: %s"
% (heap_base))
vheap = SizedVirtualHeap(
heap_base,
heap_height,
blocks_per_bucket=blocks_per_bucket)
user_header_data = kwds.pop('header_data', bytes())
if type(user_header_data) is not bytes:
raise TypeError(
"'header_data' must be of type bytes. "
"Invalid type: %s" % (type(user_header_data)))
kwds['header_data'] = \
struct.pack(cls._header_struct_string,
heap_base,
heap_height,
blocks_per_bucket) + \
user_header_data
return EncryptedHeapStorage(
EncryptedBlockStorage.setup(
storage_name,
vheap.blocks_per_bucket * block_size,
vheap.bucket_count(),
**kwds))
def test_update_header_data(self):
fname = ".".join(self.id().split(".")[1:])
fname += ".bin"
fname = os.path.join(thisdir, fname)
if os.path.exists(fname):
os.remove(fname) # pragma: no cover
bsize = 10
bcount = 11
header_data = bytes(bytearray([0,1,2]))
fsetup = EncryptedBlockStorage.setup(
fname,
block_size=bsize,
block_count=bcount,
key=self._test_key,
key_size=self._test_key_size,
header_data=header_data)
fsetup.close()
new_header_data = bytes(bytearray([1,1,1]))
with EncryptedBlockStorage(fname,
key=fsetup.key,
storage_type=self._type_name) as f:
self.assertEqual(f.header_data, header_data)
f.update_header_data(new_header_data)
self.assertEqual(f.header_data, new_header_data)
with EncryptedBlockStorage(fname,
key=fsetup.key,
storage_type=self._type_name) as f:
self.assertEqual(f.header_data, new_header_data)
with self.assertRaises(ValueError):
with EncryptedBlockStorage(fname,
key=fsetup.key,
storage_type=self._type_name) as f:
f.update_header_data(bytes(bytearray([1,1])))
with self.assertRaises(ValueError):
with EncryptedBlockStorage(fname,
key=fsetup.key,
storage_type=self._type_name) as f:
f.update_header_data(bytes(bytearray([1,1,1,1])))
with EncryptedBlockStorage(fname,
key=fsetup.key,
storage_type=self._type_name) as f:
self.assertEqual(f.header_data, new_header_data)
os.remove(fname)
def setup(cls,
storage_name,
block_size,
heap_height,
blocks_per_bucket=1,
heap_base=2,
**kwds):
if 'block_count' in kwds:
raise ValueError("'block_count' keyword is not accepted")
if heap_height < 0:
raise ValueError(
"heap height must be 0 or greater. Invalid value: %s" %
(heap_height))
if blocks_per_bucket < 1:
raise ValueError("blocks_per_bucket must be 1 or greater. "
"Invalid value: %s" % (blocks_per_bucket))
if heap_base < 2:
raise ValueError(
"heap base must be 2 or greater. Invalid value: %s" %
(heap_base))
vheap = SizedVirtualHeap(heap_base,
heap_height,
blocks_per_bucket=blocks_per_bucket)
user_header_data = kwds.pop('header_data', bytes())
if type(user_header_data) is not bytes:
raise TypeError("'header_data' must be of type bytes. "
"Invalid type: %s" % (type(user_header_data)))
kwds['header_data'] = \
struct.pack(cls._header_struct_string,
heap_base,
heap_height,
blocks_per_bucket) + \
user_header_data
return EncryptedHeapStorage(
EncryptedBlockStorage.setup(storage_name,
vheap.blocks_per_bucket * block_size,
vheap.bucket_count(), **kwds))
def test_setup_fails(self):
self.assertEqual(os.path.exists(self._dummy_name), False)
with self.assertRaises(IOError):
EncryptedBlockStorage.setup(
os.path.join(thisdir,
"baselines",
"exists.empty"),
block_size=10,
block_count=10,
key=self._test_key,
key_size=self._test_key_size,
aes_mode=self._aes_mode,
storage_type=self._type_name)
self.assertEqual(os.path.exists(self._dummy_name), False)
with self.assertRaises(IOError):
EncryptedBlockStorage.setup(
os.path.join(thisdir,
"baselines",
"exists.empty"),
block_size=10,
block_count=10,
key=self._test_key,
key_size=self._test_key_size,
storage_type=self._type_name,
aes_mode=self._aes_mode,
ignore_existing=False)
self.assertEqual(os.path.exists(self._dummy_name), False)
with self.assertRaises(ValueError):
EncryptedBlockStorage.setup(
self._dummy_name,
block_size=0,
block_count=1,
key=self._test_key,
key_size=self._test_key_size,
aes_mode=self._aes_mode,
storage_type=self._type_name)
self.assertEqual(os.path.exists(self._dummy_name), False)
with self.assertRaises(ValueError):
EncryptedBlockStorage.setup(
self._dummy_name,
block_size=1,
block_count=0,
key=self._test_key,
key_size=self._test_key_size,
aes_mode=self._aes_mode,
storage_type=self._type_name)
self.assertEqual(os.path.exists(self._dummy_name), False)
with self.assertRaises(TypeError):
EncryptedBlockStorage.setup(
self._dummy_name,
block_size=1,
block_count=1,
key=self._test_key,
key_size=self._test_key_size,
aes_mode=self._aes_mode,
storage_type=self._type_name,
header_data=2)
self.assertEqual(os.path.exists(self._dummy_name), False)
with self.assertRaises(ValueError):
EncryptedBlockStorage.setup(
self._dummy_name,
block_size=1,
block_count=1,
key=self._test_key,
key_size=self._test_key_size,
aes_mode=None,
storage_type=self._type_name)
self.assertEqual(os.path.exists(self._dummy_name), False)
with self.assertRaises(ValueError):
EncryptedBlockStorage.setup(
self._dummy_name,
block_size=1,
block_count=1,
key_size=-1,
aes_mode=self._aes_mode,
storage_type=self._type_name)
self.assertEqual(os.path.exists(self._dummy_name), False)
with self.assertRaises(TypeError):
EncryptedBlockStorage.setup(
self._dummy_name,
block_size=1,
block_count=1,
key=-1,
aes_mode=self._aes_mode,
storage_type=self._type_name)
self.assertEqual(os.path.exists(self._dummy_name), False)
with self.assertRaises(ValueError):
EncryptedBlockStorage.setup(
self._dummy_name,
block_size=1,
block_count=1,
key=AES.KeyGen(AES.key_sizes[0]),
key_size=AES.key_sizes[0],
aes_mode=self._aes_mode,
storage_type=self._type_name)
self.assertEqual(os.path.exists(self._dummy_name), False)
with self.assertRaises(ValueError):
EncryptedBlockStorage.setup(
self._dummy_name,
block_size=1,
block_count=1,
key=os.urandom(AES.key_sizes[0]+100),
aes_mode=self._aes_mode,
storage_type=self._type_name)
def test_setup_withdata(self):
fname = ".".join(self.id().split(".")[1:])
fname += ".bin"
fname = os.path.join(thisdir, fname)
if os.path.exists(fname):
os.remove(fname) # pragma: no cover
bsize = 10
bcount = 11
header_data = bytes(bytearray([0,1,2]))
fsetup = EncryptedBlockStorage.setup(
fname,
block_size=bsize,
block_count=bcount,
key=self._test_key,
key_size=self._test_key_size,
aes_mode=self._aes_mode,
storage_type=self._type_name,
header_data=header_data)
fsetup.close()
self.assertEqual(type(fsetup.raw_storage),
BlockStorageTypeFactory(self._type_name))
with open(fname, 'rb') as f:
flen = len(f.read())
self.assertEqual(
flen,
EncryptedBlockStorage.compute_storage_size(
bsize,
bcount,
aes_mode=self._aes_mode,
storage_type=self._type_name,
header_data=header_data))
self.assertTrue(len(header_data) > 0)
self.assertEqual(
EncryptedBlockStorage.compute_storage_size(
bsize,
bcount,
aes_mode=self._aes_mode,
storage_type=self._type_name) <
EncryptedBlockStorage.compute_storage_size(
bsize,
bcount,
aes_mode=self._aes_mode,
storage_type=self._type_name,
header_data=header_data),
True)
self.assertEqual(
flen >
EncryptedBlockStorage.compute_storage_size(
bsize,
bcount,
aes_mode=self._aes_mode,
storage_type=self._type_name,
header_data=header_data,
ignore_header=True),
True)
with EncryptedBlockStorage(fname,
key=fsetup.key,
storage_type=self._type_name) as f:
self.assertEqual(f.header_data, header_data)
self.assertEqual(fsetup.header_data, header_data)
self.assertEqual(f.key, fsetup.key)
self.assertEqual(f.block_size, bsize)
self.assertEqual(fsetup.block_size, bsize)
self.assertEqual(f.block_count, bcount)
self.assertEqual(fsetup.block_count, bcount)
self.assertEqual(f.storage_name, fname)
self.assertEqual(fsetup.storage_name, fname)
os.remove(fname)