def compute_storage_size(cls, block_size, heap_height, blocks_per_bucket=1, heap_base=2, ignore_header=False, **kwds): assert (block_size > 0) and (block_size == int(block_size)) assert heap_height >= 0 assert blocks_per_bucket >= 1 assert heap_base >= 2 assert 'block_count' not in kwds vheap = SizedVirtualHeap(heap_base, heap_height, blocks_per_bucket=blocks_per_bucket) if ignore_header: return EncryptedBlockStorage.compute_storage_size( vheap.blocks_per_bucket * block_size, vheap.bucket_count(), ignore_header=True, **kwds) else: return cls._header_offset + \ EncryptedBlockStorage.compute_storage_size( vheap.blocks_per_bucket * block_size, vheap.bucket_count(), ignore_header=False, **kwds)
def compute_storage_size(cls, block_size, heap_height, blocks_per_bucket=1, heap_base=2, ignore_header=False, **kwds): assert (block_size > 0) and (block_size == int(block_size)) assert heap_height >= 0 assert blocks_per_bucket >= 1 assert heap_base >= 2 assert 'block_count' not in kwds vheap = SizedVirtualHeap( heap_base, heap_height, blocks_per_bucket=blocks_per_bucket) if ignore_header: return EncryptedBlockStorage.compute_storage_size( vheap.blocks_per_bucket * block_size, vheap.bucket_count(), ignore_header=True, **kwds) else: return cls._header_offset + \ EncryptedBlockStorage.compute_storage_size( vheap.blocks_per_bucket * block_size, vheap.bucket_count(), ignore_header=False, **kwds)
def test_setup(self): fname = ".".join(self.id().split(".")[1:]) fname += ".bin" fname = os.path.join(thisdir, fname) if os.path.exists(fname): os.remove(fname) # pragma: no cover bsize = 10 bcount = 11 fsetup = EncryptedBlockStorage.setup( fname, bsize, bcount, key=self._test_key, key_size=self._test_key_size, aes_mode=self._aes_mode, storage_type=self._type_name) fsetup.close() self.assertEqual(type(fsetup.raw_storage), BlockStorageTypeFactory(self._type_name)) with open(fname, 'rb') as f: flen = len(f.read()) self.assertEqual( flen, EncryptedBlockStorage.compute_storage_size( bsize, bcount, aes_mode=self._aes_mode, storage_type=self._type_name)) self.assertEqual( flen > EncryptedBlockStorage.compute_storage_size( bsize, bcount, aes_mode=self._aes_mode, storage_type=self._type_name, ignore_header=True), True) with EncryptedBlockStorage(fname, key=fsetup.key, storage_type=self._type_name) as f: self.assertEqual(f.header_data, bytes()) self.assertEqual(fsetup.header_data, bytes()) self.assertEqual(f.key, fsetup.key) self.assertEqual(f.block_size, bsize) self.assertEqual(fsetup.block_size, bsize) self.assertEqual(f.block_count, bcount) self.assertEqual(fsetup.block_count, bcount) self.assertEqual(f.storage_name, fname) self.assertEqual(fsetup.storage_name, fname) # tamper with the plaintext index with open(fname, 'r+b') as f: f.seek(0) f.write(struct.pack("!L",0)) with self.assertRaises(ValueError): with EncryptedBlockStorage(fname, key=fsetup.key, storage_type=self._type_name) as f: pass # pragma: no cover os.remove(fname)
def main(): print("Storage Name: %s" % (storage_name)) print("Block Count: %s" % (block_count)) print("Block Size: %s" % (MemorySize(block_size))) print("Total Memory: %s" % (MemorySize(block_size*block_count))) print("Actual Storage Required: %s" % (MemorySize( EncryptedBlockStorage.compute_storage_size( block_size, block_count, storage_type='sftp')))) print("") # Start an SSH client using paramiko print("Starting SSH Client") with paramiko.SSHClient() as ssh: ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.load_system_host_keys() ssh.connect(ssh_host, username=ssh_username, password=ssh_password) print("Setting Up Encrypted Block Storage") setup_start = time.time() with EncryptedBlockStorage.setup(storage_name, block_size, block_count, storage_type='sftp', sshclient=ssh, ignore_existing=True) as f: print("Total Setup Time: %2.f s" % (time.time()-setup_start)) print("Total Data Transmission: %s" % (MemorySize(f.bytes_sent + f.bytes_received))) print("") # We close the device and reopen it after # setup to reset the bytes sent and bytes # received stats. with EncryptedBlockStorage(storage_name, key=f.key, storage_type='sftp', sshclient=ssh) as f: test_count = 1000 start_time = time.time() for t in tqdm.tqdm(list(range(test_count)), desc="Running I/O Performance Test"): f.read_block(random.randint(0,f.block_count-1)) stop_time = time.time() print("Access Block Avg. Data Transmitted: %s (%.3fx)" % (MemorySize((f.bytes_sent + f.bytes_received)/float(test_count)), (f.bytes_sent + f.bytes_received)/float(test_count)/float(block_size))) print("Access Block Avg. Latency: %.2f ms" % ((stop_time-start_time)/float(test_count)*1000)) print("")
def main(): print("Storage Name: %s" % (storage_name)) print("Block Count: %s" % (block_count)) print("Block Size: %s" % (MemorySize(block_size))) print("Total Memory: %s" % (MemorySize(block_size*block_count))) print("Actual Storage Required: %s" % (MemorySize( EncryptedBlockStorage.compute_storage_size( block_size, block_count, storage_type='ram')))) print("") print("Setting Up Encrypted Block Storage") setup_start = time.time() with EncryptedBlockStorage.setup(storage_name, # RAM storage ignores this argument block_size, block_count, storage_type='ram', ignore_existing=True) as f: print("Total Setup Time: %2.f s" % (time.time()-setup_start)) print("Total Data Transmission: %s" % (MemorySize(f.bytes_sent + f.bytes_received))) print("") # This must be done after closing the file to ensure the lock flag # is set to False in the saved data. The tofile method only exists # on BlockStorageRAM f.raw_storage.tofile(storage_name) # We close the device and reopen it after # setup to reset the bytes sent and bytes # received stats. with EncryptedBlockStorage(BlockStorageRAM.fromfile(storage_name), key=f.key) as f: test_count = 1000 start_time = time.time() for t in tqdm.tqdm(list(range(test_count)), desc="Running I/O Performance Test"): f.read_block(random.randint(0,f.block_count-1)) stop_time = time.time() print("Access Block Avg. Data Transmitted: %s (%.3fx)" % (MemorySize((f.bytes_sent + f.bytes_received)/float(test_count)), (f.bytes_sent + f.bytes_received)/float(test_count)/float(block_size))) print("Access Block Avg. Latency: %.2f ms" % ((stop_time-start_time)/float(test_count)*1000)) print("") # cleanup because this is a test example os.remove(storage_name)
def main(): print("Storage Name: %s" % (storage_name)) print("Block Count: %s" % (block_count)) print("Block Size: %s" % (MemorySize(block_size))) print("Total Memory: %s" % (MemorySize(block_size * block_count))) print("Actual Storage Required: %s" % (MemorySize( EncryptedBlockStorage.compute_storage_size( block_size, block_count, storage_type='sftp')))) print("") # Start an SSH client using paramiko print("Starting SSH Client") with paramiko.SSHClient() as ssh: ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.load_system_host_keys() ssh.connect(ssh_host, username=ssh_username, password=ssh_password) print("Setting Up Encrypted Block Storage") setup_start = time.time() with EncryptedBlockStorage.setup(storage_name, block_size, block_count, storage_type='sftp', sshclient=ssh, ignore_existing=True) as f: print("Total Setup Time: %2.f s" % (time.time() - setup_start)) print("Total Data Transmission: %s" % (MemorySize(f.bytes_sent + f.bytes_received))) print("") # We close the device and reopen it after # setup to reset the bytes sent and bytes # received stats. with EncryptedBlockStorage(storage_name, key=f.key, storage_type='sftp', sshclient=ssh) as f: test_count = 1000 start_time = time.time() for t in tqdm.tqdm(list(range(test_count)), desc="Running I/O Performance Test"): f.read_block(random.randint(0, f.block_count - 1)) stop_time = time.time() print("Access Block Avg. Data Transmitted: %s (%.3fx)" % (MemorySize( (f.bytes_sent + f.bytes_received) / float(test_count)), (f.bytes_sent + f.bytes_received) / float(test_count) / float(block_size))) print("Access Block Avg. Latency: %.2f ms" % ((stop_time - start_time) / float(test_count) * 1000)) print("")
def main(): print("Storage Name: %s" % (storage_name)) print("Block Count: %s" % (block_count)) print("Block Size: %s" % (MemorySize(block_size))) print("Total Memory: %s" % (MemorySize(block_size*block_count))) print("Actual Storage Required: %s" % (MemorySize( EncryptedBlockStorage.compute_storage_size( block_size, block_count, storage_type='mmap')))) print("") print("Setting Up Encrypted Block Storage") setup_start = time.time() with EncryptedBlockStorage.setup(storage_name, block_size, block_count, storage_type='mmap', ignore_existing=True) as f: print("Total Setup Time: %2.f s" % (time.time()-setup_start)) print("Total Data Transmission: %s" % (MemorySize(f.bytes_sent + f.bytes_received))) print("") # We close the device and reopen it after # setup to reset the bytes sent and bytes # received stats. with EncryptedBlockStorage(storage_name, key=f.key, storage_type='mmap') as f: test_count = 1000 start_time = time.time() for t in tqdm.tqdm(list(range(test_count)), desc="Running I/O Performance Test"): f.read_block(random.randint(0,f.block_count-1)) stop_time = time.time() print("Access Block Avg. Data Transmitted: %s (%.3fx)" % (MemorySize((f.bytes_sent + f.bytes_received)/float(test_count)), (f.bytes_sent + f.bytes_received)/float(test_count)/float(block_size))) print("Access Block Avg. Latency: %.2f ms" % ((stop_time-start_time)/float(test_count)*1000)) print("") # cleanup because this is a test example os.remove(storage_name)
def setUpClass(cls): assert cls._type_name is not None assert cls._aes_mode is not None assert not ((cls._test_key is not None) and \ (cls._test_key_size is not None)) fd, cls._dummy_name = tempfile.mkstemp() os.close(fd) try: os.remove(cls._dummy_name) except OSError: # pragma: no cover pass # pragma: no cover cls._block_size = 25 cls._block_count = 5 cls._testfname = cls.__name__ + "_testfile.bin" cls._blocks = [] f = EncryptedBlockStorage.setup( cls._testfname, cls._block_size, cls._block_count, key_size=cls._test_key_size, key=cls._test_key, storage_type=cls._type_name, aes_mode=cls._aes_mode, initialize=lambda i: bytes(bytearray([i])*cls._block_size), ignore_existing=True) f.close() cls._key = f.key for i in range(cls._block_count): data = bytearray([i])*cls._block_size cls._blocks.append(data)
def main(): print("Storage Name: %s" % (storage_name)) print("Block Count: %s" % (block_count)) print("Block Size: %s" % (MemorySize(block_size))) print("Total Memory: %s" % (MemorySize(block_size * block_count))) print("Actual Storage Required: %s" % (MemorySize( EncryptedBlockStorage.compute_storage_size( block_size, block_count, storage_type='s3')))) print("") print("Setting Up Encrypted Block Storage") setup_start = time.time() with EncryptedBlockStorage.setup(storage_name, block_size, block_count, storage_type='s3', bucket_name=bucket_name, ignore_existing=True) as f: print("Total Setup Time: %2.f s" % (time.time() - setup_start)) print("Total Data Transmission: %s" % (MemorySize(f.bytes_sent + f.bytes_received))) print("") # We close the device and reopen it after # setup to reset the bytes sent and bytes # received stats. with EncryptedBlockStorage(storage_name, key=f.key, storage_type='s3', bucket_name=bucket_name) as f: test_count = 1000 start_time = time.time() for t in tqdm.tqdm(list(range(test_count)), desc="Running I/O Performance Test"): f.read_block(random.randint(0, f.block_count - 1)) stop_time = time.time() print("Access Block Avg. Data Transmitted: %s (%.3fx)" % (MemorySize( (f.bytes_sent + f.bytes_received) / float(test_count)), (f.bytes_sent + f.bytes_received) / float(test_count) / float(block_size))) print("Access Block Avg. Latency: %.2f ms" % ((stop_time - start_time) / float(test_count) * 1000)) print("")
def __init__(self, storage, **kwds): if isinstance(storage, EncryptedBlockStorageInterface): if len(kwds): raise ValueError("Keywords not used when initializing " "with a storage device: %s" % (str(kwds))) else: storage = EncryptedBlockStorage(storage, **kwds) super(EncryptedHeapStorage, self).__init__(storage)
def test_init_exists(self): self.assertEqual(os.path.exists(self._testfname), True) with EncryptedBlockStorage(self._testfname, key=self._key, storage_type=self._storage_type) as f: databefore = f.read_blocks(list(range(f.block_count))) with self.assertRaises(ValueError): with EncryptedBlockStorage(self._testfname, key=self._key, storage_type=self._storage_type) as fb: with TopCachedEncryptedHeapStorage( EncryptedHeapStorage(fb, key=self._key), **self._init_kwds) as f: pass # pragma: no cover with TopCachedEncryptedHeapStorage( EncryptedHeapStorage( self._testfname, key=self._key, storage_type=self._storage_type), **self._init_kwds) as f: self.assertEqual(f.key, self._key) self.assertEqual(f.bucket_size, self._block_size * \ self._blocks_per_bucket) self.assertEqual(f.bucket_count, self._bucket_count) self.assertEqual(f.storage_name, self._testfname) self.assertEqual(f.header_data, bytes()) self.assertEqual(os.path.exists(self._testfname), True) with TopCachedEncryptedHeapStorage( EncryptedHeapStorage( self._testfname, key=self._key, storage_type=self._storage_type), **self._init_kwds) as f: dataafter = f.bucket_storage.read_blocks( list(range(f.bucket_storage.block_count))) self.assertEqual(databefore, dataafter)
def setup(cls, storage_name, block_size, heap_height, blocks_per_bucket=1, heap_base=2, **kwds): if 'block_count' in kwds: raise ValueError("'block_count' keyword is not accepted") if heap_height < 0: raise ValueError( "heap height must be 0 or greater. Invalid value: %s" % (heap_height)) if blocks_per_bucket < 1: raise ValueError( "blocks_per_bucket must be 1 or greater. " "Invalid value: %s" % (blocks_per_bucket)) if heap_base < 2: raise ValueError( "heap base must be 2 or greater. Invalid value: %s" % (heap_base)) vheap = SizedVirtualHeap( heap_base, heap_height, blocks_per_bucket=blocks_per_bucket) user_header_data = kwds.pop('header_data', bytes()) if type(user_header_data) is not bytes: raise TypeError( "'header_data' must be of type bytes. " "Invalid type: %s" % (type(user_header_data))) kwds['header_data'] = \ struct.pack(cls._header_struct_string, heap_base, heap_height, blocks_per_bucket) + \ user_header_data return EncryptedHeapStorage( EncryptedBlockStorage.setup( storage_name, vheap.blocks_per_bucket * block_size, vheap.bucket_count(), **kwds))
def test_update_header_data(self): fname = ".".join(self.id().split(".")[1:]) fname += ".bin" fname = os.path.join(thisdir, fname) if os.path.exists(fname): os.remove(fname) # pragma: no cover bsize = 10 bcount = 11 header_data = bytes(bytearray([0,1,2])) fsetup = EncryptedBlockStorage.setup( fname, block_size=bsize, block_count=bcount, key=self._test_key, key_size=self._test_key_size, header_data=header_data) fsetup.close() new_header_data = bytes(bytearray([1,1,1])) with EncryptedBlockStorage(fname, key=fsetup.key, storage_type=self._type_name) as f: self.assertEqual(f.header_data, header_data) f.update_header_data(new_header_data) self.assertEqual(f.header_data, new_header_data) with EncryptedBlockStorage(fname, key=fsetup.key, storage_type=self._type_name) as f: self.assertEqual(f.header_data, new_header_data) with self.assertRaises(ValueError): with EncryptedBlockStorage(fname, key=fsetup.key, storage_type=self._type_name) as f: f.update_header_data(bytes(bytearray([1,1]))) with self.assertRaises(ValueError): with EncryptedBlockStorage(fname, key=fsetup.key, storage_type=self._type_name) as f: f.update_header_data(bytes(bytearray([1,1,1,1]))) with EncryptedBlockStorage(fname, key=fsetup.key, storage_type=self._type_name) as f: self.assertEqual(f.header_data, new_header_data) os.remove(fname)
def setup(cls, storage_name, block_size, heap_height, blocks_per_bucket=1, heap_base=2, **kwds): if 'block_count' in kwds: raise ValueError("'block_count' keyword is not accepted") if heap_height < 0: raise ValueError( "heap height must be 0 or greater. Invalid value: %s" % (heap_height)) if blocks_per_bucket < 1: raise ValueError("blocks_per_bucket must be 1 or greater. " "Invalid value: %s" % (blocks_per_bucket)) if heap_base < 2: raise ValueError( "heap base must be 2 or greater. Invalid value: %s" % (heap_base)) vheap = SizedVirtualHeap(heap_base, heap_height, blocks_per_bucket=blocks_per_bucket) user_header_data = kwds.pop('header_data', bytes()) if type(user_header_data) is not bytes: raise TypeError("'header_data' must be of type bytes. " "Invalid type: %s" % (type(user_header_data))) kwds['header_data'] = \ struct.pack(cls._header_struct_string, heap_base, heap_height, blocks_per_bucket) + \ user_header_data return EncryptedHeapStorage( EncryptedBlockStorage.setup(storage_name, vheap.blocks_per_bucket * block_size, vheap.bucket_count(), **kwds))
def test_setup_fails(self): self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(IOError): EncryptedBlockStorage.setup( os.path.join(thisdir, "baselines", "exists.empty"), block_size=10, block_count=10, key=self._test_key, key_size=self._test_key_size, aes_mode=self._aes_mode, storage_type=self._type_name) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(IOError): EncryptedBlockStorage.setup( os.path.join(thisdir, "baselines", "exists.empty"), block_size=10, block_count=10, key=self._test_key, key_size=self._test_key_size, storage_type=self._type_name, aes_mode=self._aes_mode, ignore_existing=False) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(ValueError): EncryptedBlockStorage.setup( self._dummy_name, block_size=0, block_count=1, key=self._test_key, key_size=self._test_key_size, aes_mode=self._aes_mode, storage_type=self._type_name) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(ValueError): EncryptedBlockStorage.setup( self._dummy_name, block_size=1, block_count=0, key=self._test_key, key_size=self._test_key_size, aes_mode=self._aes_mode, storage_type=self._type_name) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(TypeError): EncryptedBlockStorage.setup( self._dummy_name, block_size=1, block_count=1, key=self._test_key, key_size=self._test_key_size, aes_mode=self._aes_mode, storage_type=self._type_name, header_data=2) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(ValueError): EncryptedBlockStorage.setup( self._dummy_name, block_size=1, block_count=1, key=self._test_key, key_size=self._test_key_size, aes_mode=None, storage_type=self._type_name) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(ValueError): EncryptedBlockStorage.setup( self._dummy_name, block_size=1, block_count=1, key_size=-1, aes_mode=self._aes_mode, storage_type=self._type_name) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(TypeError): EncryptedBlockStorage.setup( self._dummy_name, block_size=1, block_count=1, key=-1, aes_mode=self._aes_mode, storage_type=self._type_name) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(ValueError): EncryptedBlockStorage.setup( self._dummy_name, block_size=1, block_count=1, key=AES.KeyGen(AES.key_sizes[0]), key_size=AES.key_sizes[0], aes_mode=self._aes_mode, storage_type=self._type_name) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(ValueError): EncryptedBlockStorage.setup( self._dummy_name, block_size=1, block_count=1, key=os.urandom(AES.key_sizes[0]+100), aes_mode=self._aes_mode, storage_type=self._type_name)
def test_setup_withdata(self): fname = ".".join(self.id().split(".")[1:]) fname += ".bin" fname = os.path.join(thisdir, fname) if os.path.exists(fname): os.remove(fname) # pragma: no cover bsize = 10 bcount = 11 header_data = bytes(bytearray([0,1,2])) fsetup = EncryptedBlockStorage.setup( fname, block_size=bsize, block_count=bcount, key=self._test_key, key_size=self._test_key_size, aes_mode=self._aes_mode, storage_type=self._type_name, header_data=header_data) fsetup.close() self.assertEqual(type(fsetup.raw_storage), BlockStorageTypeFactory(self._type_name)) with open(fname, 'rb') as f: flen = len(f.read()) self.assertEqual( flen, EncryptedBlockStorage.compute_storage_size( bsize, bcount, aes_mode=self._aes_mode, storage_type=self._type_name, header_data=header_data)) self.assertTrue(len(header_data) > 0) self.assertEqual( EncryptedBlockStorage.compute_storage_size( bsize, bcount, aes_mode=self._aes_mode, storage_type=self._type_name) < EncryptedBlockStorage.compute_storage_size( bsize, bcount, aes_mode=self._aes_mode, storage_type=self._type_name, header_data=header_data), True) self.assertEqual( flen > EncryptedBlockStorage.compute_storage_size( bsize, bcount, aes_mode=self._aes_mode, storage_type=self._type_name, header_data=header_data, ignore_header=True), True) with EncryptedBlockStorage(fname, key=fsetup.key, storage_type=self._type_name) as f: self.assertEqual(f.header_data, header_data) self.assertEqual(fsetup.header_data, header_data) self.assertEqual(f.key, fsetup.key) self.assertEqual(f.block_size, bsize) self.assertEqual(fsetup.block_size, bsize) self.assertEqual(f.block_count, bcount) self.assertEqual(fsetup.block_count, bcount) self.assertEqual(f.storage_name, fname) self.assertEqual(fsetup.storage_name, fname) os.remove(fname)