def main(): print("Storage Name: %s" % (storage_name)) print("Block Count: %s" % (block_count)) print("Block Size: %s" % (MemorySize(block_size))) print("Total Memory: %s" % (MemorySize(block_size * block_count))) print("Actual Storage Required: %s" % (MemorySize( PathORAM.compute_storage_size( block_size, block_count, storage_type='mmap')))) print("") print("Setting Up Path ORAM Storage Locally") setup_start = time.time() with PathORAM.setup(storage_name, block_size, block_count, storage_type='mmap', ignore_existing=True) as f: print("Total Setup Time: %.2f s" % (time.time() - setup_start)) print("Current Stash Size: %s" % len(f.stash)) print("Total Data Transmission: %s" % (MemorySize(f.bytes_sent + f.bytes_received))) print("") print("Saving key to file: %s.key" % (storage_name)) save_private_key(storage_name + ".key", f.key) print("Saving stash to file: %s.stash" % (storage_name)) with open(storage_name + ".stash", 'wb') as fstash: pickle.dump(f.stash, fstash) print("Saving position map to file: %s.position" % (storage_name)) with open(storage_name + ".position", 'wb') as fpos: pickle.dump(f.position_map, fpos) # Start an SSH client using paramiko print("Starting SSH Client") with paramiko.SSHClient() as ssh: ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.load_system_host_keys() ssh.connect(ssh_host, username=ssh_username, password=ssh_password) sftp = ssh.open_sftp() def my_hook(t): def inner(b, total): t.total = total t.update(b - inner.last_b) inner.last_b = b inner.last_b = 0 return inner with tqdm.tqdm(desc="Transferring Storage", unit='B', unit_scale=True, miniters=1) as t: sftp.put(storage_name, storage_name, callback=my_hook(t)) sftp.close() print("Deleting Local Copy of Storage") os.remove(storage_name)
def main(): print("Storage Name: %s" % (storage_name)) print("Block Count: %s" % (block_count)) print("Block Size: %s" % (MemorySize(block_size))) print("Total Memory: %s" % (MemorySize(block_size*block_count))) print("Actual Storage Required: %s" % (MemorySize( PathORAM.compute_storage_size( block_size, block_count, storage_type='ram')))) print("") print("Setting Up Path ORAM Storage") setup_start = time.time() with PathORAM.setup(storage_name, # RAM storage ignores this argument block_size, block_count, storage_type='ram', ignore_existing=True) as f: print("Total Setup Time: %2.f s" % (time.time()-setup_start)) print("Current Stash Size: %s" % len(f.stash)) print("Total Data Transmission: %s" % (MemorySize(f.bytes_sent + f.bytes_received))) print("") # This must be done after closing the file to ensure the lock flag # is set to False in the saved data. The tofile method only exists # on BlockStorageRAM f.raw_storage.tofile(storage_name) # We close the device and reopen it after # setup to reset the bytes sent and bytes # received stats. with PathORAM(BlockStorageRAM.fromfile(storage_name), f.stash, f.position_map, key=f.key) as f: test_count = 100 start_time = time.time() for t in tqdm.tqdm(list(range(test_count)), desc="Running I/O Performance Test"): f.read_block(random.randint(0,f.block_count-1)) stop_time = time.time() print("Current Stash Size: %s" % len(f.stash)) print("Access Block Avg. Data Transmitted: %s (%.3fx)" % (MemorySize((f.bytes_sent + f.bytes_received)/float(test_count)), (f.bytes_sent + f.bytes_received)/float(test_count)/float(block_size))) print("Access Block Avg. Latency: %.2f ms" % ((stop_time-start_time)/float(test_count)*1000)) print("") # cleanup because this is a test example os.remove(storage_name)
def main(): print("Storage Name: %s" % (storage_name)) print("Block Count: %s" % (block_count)) print("Block Size: %s" % (MemorySize(block_size))) print("Total Memory: %s" % (MemorySize(block_size * block_count))) print("Actual Storage Required: %s" % (MemorySize( PathORAM.compute_storage_size( block_size, block_count, storage_type='sftp')))) print("") # Start an SSH client using paramiko print("Starting SSH Client") with paramiko.SSHClient() as ssh: ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.load_system_host_keys() ssh.connect(ssh_host, username=ssh_username, password=ssh_password) print("Setting Up Path ORAM Storage") setup_start = time.time() with PathORAM.setup(storage_name, block_size, block_count, storage_type='sftp', sshclient=ssh, ignore_existing=True) as f: print("Total Setup Time: %.2f s" % (time.time() - setup_start)) print("Current Stash Size: %s" % len(f.stash)) print("Total Data Transmission: %s" % (MemorySize(f.bytes_sent + f.bytes_received))) print("") # We close the device and reopen it after # setup to reset the bytes sent and bytes # received stats. with PathORAM(storage_name, f.stash, f.position_map, key=f.key, storage_type='sftp', sshclient=ssh) as f: test_count = 100 start_time = time.time() for t in tqdm.tqdm(list(range(test_count)), desc="Running I/O Performance Test"): f.read_block(random.randint(0, f.block_count - 1)) stop_time = time.time() print("Current Stash Size: %s" % len(f.stash)) print("Access Block Avg. Data Transmitted: %s (%.3fx)" % (MemorySize( (f.bytes_sent + f.bytes_received) / float(test_count)), (f.bytes_sent + f.bytes_received) / float(test_count) / float(block_size))) print("Fetch Block Avg. Latency: %.2f ms" % ((stop_time - start_time) / float(test_count) * 1000)) print("")
def main(): print("Storage Name: %s" % (storage_name)) print("Block Count: %s" % (block_count)) print("Block Size: %s" % (MemorySize(block_size))) print("Total Memory: %s" % (MemorySize(block_size * block_count))) print("Actual Storage Required: %s" % (MemorySize( PathORAM.compute_storage_size( block_size, block_count, storage_type='ram')))) print("") print("Setting Up Path ORAM Storage") setup_start = time.time() with PathORAM.setup( storage_name, # RAM storage ignores this argument block_size, block_count, storage_type='ram', ignore_existing=True) as f: print("Total Setup Time: %2.f s" % (time.time() - setup_start)) print("Current Stash Size: %s" % len(f.stash)) print("Total Data Transmission: %s" % (MemorySize(f.bytes_sent + f.bytes_received))) print("") # This must be done after closing the file to ensure the lock flag # is set to False in the saved data. The tofile method only exists # on BlockStorageRAM f.raw_storage.tofile(storage_name) # We close the device and reopen it after # setup to reset the bytes sent and bytes # received stats. with PathORAM(BlockStorageRAM.fromfile(storage_name), f.stash, f.position_map, key=f.key) as f: test_count = 100 start_time = time.time() for t in tqdm.tqdm(list(range(test_count)), desc="Running I/O Performance Test"): f.read_block(random.randint(0, f.block_count - 1)) stop_time = time.time() print("Current Stash Size: %s" % len(f.stash)) print("Access Block Avg. Data Transmitted: %s (%.3fx)" % (MemorySize( (f.bytes_sent + f.bytes_received) / float(test_count)), (f.bytes_sent + f.bytes_received) / float(test_count) / float(block_size))) print("Access Block Avg. Latency: %.2f ms" % ((stop_time - start_time) / float(test_count) * 1000)) print("") # cleanup because this is a test example os.remove(storage_name)
def main(): print("Storage Name: %s" % (storage_name)) print("Block Count: %s" % (block_count)) print("Block Size: %s" % (MemorySize(block_size))) print("Total Memory: %s" % (MemorySize(block_size*block_count))) print("Actual Storage Required: %s" % (MemorySize( PathORAM.compute_storage_size( block_size, block_count, storage_type='s3')))) print("") print("Setting Up Path ORAM Storage") setup_start = time.time() with PathORAM.setup(storage_name, block_size, block_count, storage_type='s3', bucket_name=bucket_name, ignore_existing=True) as f: print("Total Setup Time: %.2f s" % (time.time()-setup_start)) print("Current Stash Size: %s" % len(f.stash)) print("Total Data Transmission: %s" % (MemorySize(f.bytes_sent + f.bytes_received))) print("") # We close the device and reopen it after # setup to reset the bytes sent and bytes # received stats. with PathORAM(storage_name, f.stash, f.position_map, key=f.key, storage_type='s3', bucket_name=bucket_name) as f: test_count = 100 start_time = time.time() for t in tqdm.tqdm(list(range(test_count)), desc="Running I/O Performance Test"): f.read_block(random.randint(0,f.block_count-1)) stop_time = time.time() print("Current Stash Size: %s" % len(f.stash)) print("Access Block Avg. Data Transmitted: %s (%.3fx)" % (MemorySize((f.bytes_sent + f.bytes_received)/float(test_count)), (f.bytes_sent + f.bytes_received)/float(test_count)/float(block_size))) print("Fetch Block Avg. Latency: %.2f ms" % ((stop_time-start_time)/float(test_count)*1000)) print("")
def test_write_blocks(self): data = [bytearray([self._block_count])*self._block_size for i in xrange(self._block_count)] with PathORAM(self._testfname, self._stash, self._position_map, key=self._key, storage_type=self._type_name, **self._kwds) as f: orig = f.read_blocks(list(xrange(self._block_count))) self.assertEqual(len(orig), self._block_count) for i, block in enumerate(orig): self.assertEqual(list(bytearray(block)), list(self._blocks[i])) f.write_blocks(list(xrange(self._block_count)), [bytes(b) for b in data]) new = f.read_blocks(list(xrange(self._block_count))) self.assertEqual(len(new), self._block_count) for i, block in enumerate(new): self.assertEqual(list(bytearray(block)), list(data[i])) f.write_blocks(list(xrange(self._block_count)), [bytes(b) for b in self._blocks]) orig = f.read_blocks(list(xrange(self._block_count))) self.assertEqual(len(orig), self._block_count) for i, block in enumerate(orig): self.assertEqual(list(bytearray(block)), list(self._blocks[i]))
def setUp(self): self.oram = PathORAM.setup("test", BLOCK_SIZE, BLOCK_COUNT, storage_type="ram", ignore_existing=True) self.store = privatekv.store.KVORAM(self.oram)
def main(): print("Loading key from file: %s.key" % (storage_name)) key = load_private_key(storage_name+".key") print("Loading stash from file: %s.stash" % (storage_name)) with open(storage_name+".stash", 'rb') as fstash: stash = pickle.load(fstash) print("Loading position map from file: %s.position" % (storage_name)) with open(storage_name+".position", 'rb') as fpos: position_map = pickle.load(fpos) # Start an SSH client using paramiko print("Starting SSH Client") with paramiko.SSHClient() as ssh: ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.load_system_host_keys() ssh.connect(ssh_host, username=ssh_username, password=ssh_password) with PathORAM(storage_name, stash, position_map, key=key, storage_type='sftp', cached_levels=6, concurrency_level=3, threadpool_size=multiprocessing.cpu_count()*2, sshclient=ssh) as f: try: test_count = 100 start_time = time.time() for t in tqdm.tqdm(list(range(test_count)), desc="Running I/O Performance Test"): f.read_block(random.randint(0,f.block_count-1)) stop_time = time.time() print("Current Stash Size: %s" % len(f.stash)) print("Fetch Block Avg. Latency: %.2f ms" % ((stop_time-start_time)/float(test_count)*1000)) print("") finally: print("Saving stash to file: %s.stash" % (storage_name)) with open(storage_name+".stash", 'wb') as fstash: pickle.dump(f.stash, fstash) print("Saving position map to file: %s.position" % (storage_name)) with open(storage_name+".position", 'wb') as fpos: pickle.dump(f.position_map, fpos)
def test_compute_level_load(self): oram = PathORAM.setup("test", 32, 1, bucket_capacity=1, storage_type="ram", ignore_existing=True, cached_levels=0) result = privatekv.utils.compute_avg_level_load(oram) self.assertEqual(1.0, result[0]) oram.close()
def test_init_noexists(self): self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(IOError): with PathORAM( self._dummy_name, self._stash, self._position_map, key=self._key, storage_type=self._type_name, **self._kwds) as f: pass # pragma: no cover
def test_write_block(self): data = bytearray([self._block_count])*self._block_size self.assertEqual(len(data) > 0, True) with PathORAM(self._testfname, self._stash, self._position_map, key=self._key, storage_type=self._type_name, **self._kwds) as f: for i in xrange(self._block_count): self.assertNotEqual(list(bytearray(f.read_block(i))), list(data)) for i in xrange(self._block_count): f.write_block(i, bytes(data)) for i in xrange(self._block_count): self.assertEqual(list(bytearray(f.read_block(i))), list(data)) for i, block in enumerate(self._blocks): f.write_block(i, bytes(block))
def setUpClass(cls): assert cls._type_name is not None assert cls._aes_mode is not None assert not ((cls._test_key is not None) and \ (cls._test_key_size is not None)) assert cls._bucket_capacity is not None assert cls._heap_base is not None assert cls._kwds is not None fd, cls._dummy_name = tempfile.mkstemp() os.close(fd) try: os.remove(cls._dummy_name) except OSError: # pragma: no cover pass # pragma: no cover cls._block_size = 25 cls._block_count = 47 cls._testfname = cls.__name__ + "_testfile.bin" cls._blocks = [] f = PathORAM.setup( cls._testfname, cls._block_size, cls._block_count, bucket_capacity=cls._bucket_capacity, heap_base=cls._heap_base, key_size=cls._test_key_size, key=cls._test_key, storage_type=cls._type_name, aes_mode=cls._aes_mode, initialize=lambda i: bytes(bytearray([i])*cls._block_size), ignore_existing=True, **cls._kwds) f.close() cls._key = f.key cls._stash = f.stash cls._position_map = f.position_map for i in range(cls._block_count): data = bytearray([i])*cls._block_size cls._blocks.append(data)
def test_read_blocks(self): with PathORAM(self._testfname, self._stash, self._position_map, key=self._key, storage_type=self._type_name, **self._kwds) as f: data = f.read_blocks(list(xrange(self._block_count))) self.assertEqual(len(data), self._block_count) for i, block in enumerate(data): self.assertEqual(list(bytearray(block)), list(self._blocks[i])) data = f.read_blocks([0]) self.assertEqual(len(data), 1) self.assertEqual(list(bytearray(data[0])), list(self._blocks[0])) self.assertEqual(len(self._blocks) > 1, True) data = f.read_blocks(list(xrange(1, self._block_count)) + [0]) self.assertEqual(len(data), self._block_count) for i, block in enumerate(data[:-1], 1): self.assertEqual(list(bytearray(block)), list(self._blocks[i])) self.assertEqual(list(bytearray(data[-1])), list(self._blocks[0]))
def test_load_save_client_data(self): oram = PathORAM.setup(os.path.join(self.tmpdir.name, "test"), BLOCK_SIZE, BLOCK_COUNT, storage_type="file", ignore_existing=True) data = b"hello world" data += b"\x00" * (oram.block_size - len(data)) oram.write_block(0, bytes(data)) oram.close() privatekv.utils.save_oram_client_data(oram, path=self.tmpdir.name) key, stash, pm = privatekv.utils.read_oram_client_data( "test", path=self.tmpdir.name) oram = PathORAM(os.path.join(self.tmpdir.name, "test"), stash, pm, key=key, storage_type="file") self.assertEqual(data, oram.read_block(0)) oram.close()
def randomString(stringLength=10): """Generate a random string of fixed length """ letters = string.ascii_lowercase return ''.join(random.choice(letters) for i in range(stringLength)) if __name__ == "__main__": ap = ArgumentParser(description="runs the PrivateKV S3 access logging test") ap.add_argument("-b", default="test", help="the S3 bucket to use") ap.add_argument("-i", default=192, type=int, help="number of inserts") ap.add_argument("--create", default=False, action="store_true", help="only create the oram") args = ap.parse_args() if args.create: print("Setting up ORAM") pyoram.config.SHOW_PROGRESS_BAR = True oram = PathORAM.setup("test", BLOCK_SIZE, BLOCK_COUNT, cached_levels=0, storage_type="s3", bucket_name=args.b, ignore_existing=True) kv = privatekv.store.KVORAM(oram) keys = set() inserts = args.i for i in range(inserts): key = str(uuid.uuid4()) kv.put(key, randomString()) keys.add(key) print("inserts %d / %d" % (i, inserts)) keys_file = open("keys", "w") for key in list(keys): keys_file.write(key + "\n") keys_file.close()
def main(): print("Storage Name: %s" % (storage_name)) print("Block Count: %s" % (block_count)) print("Block Size: %s" % (MemorySize(block_size))) print("Total Memory: %s" % (MemorySize(block_size*block_count))) print("Actual Storage Required: %s" % (MemorySize( PathORAM.compute_storage_size( block_size, block_count, storage_type='mmap')))) print("") print("Setting Up Path ORAM Storage Locally") setup_start = time.time() with PathORAM.setup(storage_name, block_size, block_count, storage_type='mmap', ignore_existing=True) as f: print("Total Setup Time: %.2f s" % (time.time()-setup_start)) print("Current Stash Size: %s" % len(f.stash)) print("Total Data Transmission: %s" % (MemorySize(f.bytes_sent + f.bytes_received))) print("") print("Saving key to file: %s.key" % (storage_name)) save_private_key(storage_name+".key", f.key) print("Saving stash to file: %s.stash" % (storage_name)) with open(storage_name+".stash", 'wb') as fstash: pickle.dump(f.stash, fstash) print("Saving position map to file: %s.position" % (storage_name)) with open(storage_name+".position", 'wb') as fpos: pickle.dump(f.position_map, fpos) # Start an SSH client using paramiko print("Starting SSH Client") with paramiko.SSHClient() as ssh: ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.load_system_host_keys() ssh.connect(ssh_host, username=ssh_username, password=ssh_password) sftp = ssh.open_sftp() def my_hook(t): def inner(b, total): t.total = total t.update(b - inner.last_b) inner.last_b = b inner.last_b = 0 return inner with tqdm.tqdm(desc="Transferring Storage", unit='B', unit_scale=True, miniters=1) as t: sftp.put(storage_name, storage_name, callback=my_hook(t)) sftp.close() print("Deleting Local Copy of Storage") os.remove(storage_name)
def test_init_exists(self): self.assertEqual(os.path.exists(self._testfname), True) with open(self._testfname, 'rb') as f: databefore = f.read() # no key with self.assertRaises(ValueError): with PathORAM(self._testfname, self._stash, self._position_map, storage_type=self._type_name, **self._kwds) as f: pass # pragma: no cover # stash does not match digest with self.assertRaises(ValueError): with PathORAM(self._testfname, {1: bytes()}, self._position_map, key=self._key, storage_type=self._type_name, **self._kwds) as f: pass # pragma: no cover # stash hash invalid key (negative) with self.assertRaises(ValueError): with PathORAM(self._testfname, {-1: bytes()}, self._position_map, key=self._key, storage_type=self._type_name, **self._kwds) as f: pass # pragma: no cover # position map has invalid item (negative) with self.assertRaises(ValueError): with PathORAM(self._testfname, self._stash, [-1], key=self._key, storage_type=self._type_name, **self._kwds) as f: pass # pragma: no cover # position map does not match digest with self.assertRaises(ValueError): with PathORAM(self._testfname, self._stash, [1], key=self._key, storage_type=self._type_name, **self._kwds) as f: pass # pragma: no cover with self.assertRaises(ValueError): with EncryptedHeapStorage(self._testfname, key=self._key, storage_type=self._type_name) as fb: with PathORAM(fb, self._stash, self._position_map, key=self._key, storage_type=self._type_name, **self._kwds) as f: self.assertIs(f.heap_storage, fb) pass # pragma: no cover with PathORAM(self._testfname, self._stash, self._position_map, key=self._key, storage_type=self._type_name, **self._kwds) as f: self.assertEqual(f.key, self._key) self.assertEqual(f.block_size, self._block_size) self.assertEqual(f.block_count, self._block_count) self.assertEqual(f.storage_name, self._testfname) self.assertEqual(f.header_data, bytes()) self.assertEqual(os.path.exists(self._testfname), True) with open(self._testfname, 'rb') as f: dataafter = f.read() self.assertEqual(databefore[-(self._block_count*self._block_size):], dataafter[-(self._block_count*self._block_size):])
def test_setup(self): fname = ".".join(self.id().split(".")[1:]) fname += ".bin" fname = os.path.join(thisdir, fname) if os.path.exists(fname): os.remove(fname) # pragma: no cover bsize = 10 bcount = 11 fsetup = PathORAM.setup( fname, bsize, bcount, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, key=self._test_key, key_size=self._test_key_size, aes_mode=self._aes_mode, storage_type=self._type_name, **self._kwds) fsetup.close() self.assertEqual(type(fsetup.raw_storage), BlockStorageTypeFactory(self._type_name)) # test that these can be called with default keyword values fsetup.stash_digest(fsetup.stash) fsetup.position_map_digest(fsetup.position_map) with open(fname, 'rb') as f: flen = len(f.read()) self.assertEqual( flen, PathORAM.compute_storage_size( bsize, bcount, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, aes_mode=self._aes_mode, storage_type=self._type_name)) self.assertEqual( flen > PathORAM.compute_storage_size( bsize, bcount, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, aes_mode=self._aes_mode, storage_type=self._type_name, ignore_header=True), True) with PathORAM(fname, fsetup.stash, fsetup.position_map, key=fsetup.key, storage_type=self._type_name, **self._kwds) as f: self.assertEqual(f.header_data, bytes()) self.assertEqual(fsetup.header_data, bytes()) self.assertEqual(f.key, fsetup.key) self.assertEqual(f.block_size, bsize) self.assertEqual(fsetup.block_size, bsize) self.assertEqual(f.block_count, bcount) self.assertEqual(fsetup.block_count, bcount) self.assertEqual(f.storage_name, fname) self.assertEqual(fsetup.storage_name, fname) os.remove(fname)
def randomString(stringLength=10): """Generate a random string of fixed length """ letters = string.ascii_lowercase return ''.join(random.choice(letters) for i in range(stringLength)) if __name__ == "__main__": ap = ArgumentParser(description="runs the PrivateKV S3 insertion performance test") ap.add_argument("-c", default=0, type=int, help="cached levels") ap.add_argument("-b", default="test", help="the S3 bucket to use") ap.add_argument("-i", default=2000, type=int, help="number of inserts") args = ap.parse_args() print("Setting up ORAM") pyoram.config.SHOW_PROGRESS_BAR = True oram = PathORAM.setup("test", BLOCK_SIZE, BLOCK_COUNT, cached_levels=args.c, storage_type="s3", bucket_name=args.b, ignore_existing=True) pyoram.config.SHOW_PROGRESS_BAR = False kv = privatekv.store.KVORAM(oram) output_file = open("s3_insert_time.csv", "w") inserts = args.i for i in range(inserts): key = uuid.uuid4() def do_put(): kv.put(key, randomString()) duration = timeit.timeit(do_put, number=1) output_file.write("%i,%f\n" % (i+1, duration)) print("inserts %d / %d" % (i, inserts)) output_file.close() oram.close()
keyfile_name = "%s.key" % (args.storename) stashfile_name = "%s.stash" % (args.storename) positionfile_name = "%s.position" % (args.storename) with open(keyfile_name, "rb") as keyfile: key = keyfile.read() with open(stashfile_name, "rb") as stashfile: stash = pickle.load(stashfile) with open(positionfile_name, "rb") as positionfile: position_map = pickle.load(positionfile) ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) rsa_key = paramiko.RSAKey.from_private_key_file(args.key) ssh.connect(args.host, port=args.port, username=args.user, pkey=rsa_key) sftp = ssh.open_sftp() with PathORAM(args.storename, stash, position_map, key=key, storage_type="sftp", cached_levels=6, concurrency_level=1, sshclient=ssh) as oram: oram.read_block(0) sftp.close() ssh.close()
return ''.join(random.choice(letters) for i in range(stringLength)) if __name__ == "__main__": ap = ArgumentParser( description="runs the PrivateKV local insert time test") ap.add_argument("-n", default=2**15, type=int, help="number of hash buckets") args = ap.parse_args() print("Setting up ORAM") oram = PathORAM.setup("test", BLOCK_SIZE, args.n, storage_type="ram", ignore_existing=True, cached_levels=0) kv = privatekv.store.KVORAM(oram) output_file = open("insert_time_blocks_%d.csv" % (args.n), "w") inserts = 1000000 for i in range(inserts): key = uuid.uuid4() def do_put(): kv.put(key, randomString()) duration = timeit.timeit(do_put, number=1) output_file.write("%i,%f\n" % (i + 1, duration)) if 0 == i % 1000:
ap = ArgumentParser( description="initialize a Path ORAM and store it on a SFTP server") ap.add_argument("storename", help="the store name") ap.add_argument("host", help="the host name") ap.add_argument("user", help="the user name") ap.add_argument("key", help="RSA private key file") ap.add_argument("--port", "-p", default=22) args = ap.parse_args() print("Building ORAM...") with NamedTemporaryFile(delete=False) as tmp: tmpname = tmp.name oram = PathORAM.setup(tmpname, BLOCK_SIZE, BLOCK_COUNT, storage_type="file", ignore_existing=True) oram.close() keyfile_name = "%s.key" % (args.storename) stashfile_name = "%s.stash" % (args.storename) positionfile_name = "%s.position" % (args.storename) with open(keyfile_name, "wb") as keyfile: keyfile.write(oram.key) with open(stashfile_name, "wb") as stashfile: pickle.dump(oram.stash, stashfile) with open(positionfile_name, "wb") as positionfile: pickle.dump(oram.position_map, positionfile)
def test_read_block(self): with PathORAM(self._testfname, self._stash, self._position_map, key=self._key, storage_type=self._type_name, **self._kwds) as f: for i, data in enumerate(self._blocks): self.assertEqual(list(bytearray(f.read_block(i))), list(self._blocks[i])) for i, data in enumerate(self._blocks): self.assertEqual(list(bytearray(f.read_block(i))), list(self._blocks[i])) for i, data in reversed(list(enumerate(self._blocks))): self.assertEqual(list(bytearray(f.read_block(i))), list(self._blocks[i])) for i, data in reversed(list(enumerate(self._blocks))): self.assertEqual(list(bytearray(f.read_block(i))), list(self._blocks[i])) with PathORAM(self._testfname, self._stash, self._position_map, key=self._key, storage_type=self._type_name, **self._kwds) as f: self.assertEqual(list(bytearray(f.read_block(0))), list(self._blocks[0])) self.assertEqual(list(bytearray(f.read_block(self._block_count-1))), list(self._blocks[-1])) # test eviction behavior of the tree oram helper with PathORAM(self._testfname, self._stash, self._position_map, key=self._key, storage_type=self._type_name, **self._kwds) as f: oram = f._oram vheap = oram.storage_heap.virtual_heap Z = vheap.blocks_per_bucket def _has_vacancies(level): return any(oram.path_block_ids[i] == oram.empty_block_id for i in range(level*Z, (level+1)*Z)) for i in range(len(f.position_map)): b = f.position_map[i] f.position_map[i] = vheap.random_leaf_bucket() oram.load_path(b) block = oram.extract_block_from_path(i) if block is not None: oram.stash[i] = block # track where everyone should be able to move # to, unless the bucket fills up eviction_levels = {} for id_, level in zip(oram.path_block_ids, oram.path_block_eviction_levels): eviction_levels[id_] = level for id_ in oram.stash: block_id, block_addr = \ oram.get_block_info(oram.stash[id_]) assert block_id == id_ eviction_levels[id_] = \ vheap.clib.calculate_last_common_level( vheap.k, b, block_addr) oram.push_down_path() oram.fill_path_from_stash() oram.evict_path() # check that everyone was pushed down greedily oram.load_path(b) for pos, id_ in enumerate(oram.path_block_ids): current_level = pos // Z if (id_ != oram.empty_block_id): eviction_level = eviction_levels[id_] self.assertEqual(current_level <= eviction_level, True) if current_level < eviction_level: self.assertEqual(_has_vacancies(eviction_level), False) for id_ in oram.stash: self.assertEqual( _has_vacancies(eviction_levels[id_]), False)
def test_setup_fails(self): self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(IOError): PathORAM.setup( os.path.join(thisdir, "baselines", "exists.empty"), block_size=10, block_count=10, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, key=self._test_key, key_size=self._test_key_size, aes_mode=self._aes_mode, storage_type=self._type_name, **self._kwds) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(IOError): PathORAM.setup( os.path.join(thisdir, "baselines", "exists.empty"), block_size=10, block_count=10, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, key=self._test_key, key_size=self._test_key_size, storage_type=self._type_name, aes_mode=self._aes_mode, ignore_existing=False, **self._kwds) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(ValueError): PathORAM.setup( self._dummy_name, block_size=1, block_count=0, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, key=self._test_key, key_size=self._test_key_size, aes_mode=self._aes_mode, storage_type=self._type_name, **self._kwds) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(ValueError): PathORAM.setup( self._dummy_name, block_size=0, block_count=1, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, key=self._test_key, key_size=self._test_key_size, aes_mode=self._aes_mode, storage_type=self._type_name, **self._kwds) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(TypeError): PathORAM.setup( self._dummy_name, block_size=1, block_count=1, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, key=self._test_key, key_size=self._test_key_size, aes_mode=self._aes_mode, storage_type=self._type_name, header_data=2, **self._kwds) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(ValueError): PathORAM.setup( self._dummy_name, block_size=1, block_count=1, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, key=self._test_key, key_size=self._test_key_size, aes_mode=None, storage_type=self._type_name, **self._kwds) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(ValueError): PathORAM.setup( self._dummy_name, block_size=1, block_count=1, bucket_capacity=0, heap_base=self._heap_base, key=self._test_key, key_size=self._test_key_size, aes_mode=self._aes_mode, storage_type=self._type_name, **self._kwds) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(ValueError): PathORAM.setup( self._dummy_name, block_size=1, block_count=1, bucket_capacity=self._bucket_capacity, heap_base=1, key=self._test_key, key_size=self._test_key_size, aes_mode=self._aes_mode, storage_type=self._type_name, **self._kwds) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(ValueError): PathORAM.setup( self._dummy_name, block_size=1, block_count=1, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, key_size=-1, aes_mode=self._aes_mode, storage_type=self._type_name, **self._kwds) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(TypeError): PathORAM.setup( self._dummy_name, block_size=1, block_count=1, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, key=-1, aes_mode=self._aes_mode, storage_type=self._type_name, **self._kwds) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(ValueError): PathORAM.setup( self._dummy_name, block_size=1, block_count=1, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, key=AES.KeyGen(AES.key_sizes[-1]), key_size=AES.key_sizes[-1], aes_mode=self._aes_mode, storage_type=self._type_name, **self._kwds) self.assertEqual(os.path.exists(self._dummy_name), False) with self.assertRaises(ValueError): PathORAM.setup( self._dummy_name, block_size=1, block_count=1, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, key=os.urandom(AES.key_sizes[-1]+100), aes_mode=self._aes_mode, storage_type=self._type_name, **self._kwds) with self.assertRaises(ValueError): PathORAM.setup( self._dummy_name, block_size=1, block_count=1, heap_height=1, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, key=self._key, aes_mode=self._aes_mode, storage_type=self._type_name, **self._kwds)
def test_setup_withdata(self): fname = ".".join(self.id().split(".")[1:]) fname += ".bin" fname = os.path.join(thisdir, fname) if os.path.exists(fname): os.remove(fname) # pragma: no cover bsize = 10 bcount = 11 header_data = bytes(bytearray([0,1,2])) fsetup = PathORAM.setup( fname, block_size=bsize, block_count=bcount, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, key=self._test_key, key_size=self._test_key_size, aes_mode=self._aes_mode, storage_type=self._type_name, header_data=header_data, **self._kwds) fsetup.close() self.assertEqual(type(fsetup.raw_storage), BlockStorageTypeFactory(self._type_name)) with open(fname, 'rb') as f: flen = len(f.read()) self.assertEqual( flen, PathORAM.compute_storage_size( bsize, bcount, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, aes_mode=self._aes_mode, storage_type=self._type_name, header_data=header_data)) self.assertTrue(len(header_data) > 0) self.assertEqual( PathORAM.compute_storage_size( bsize, bcount, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, aes_mode=self._aes_mode, storage_type=self._type_name) < PathORAM.compute_storage_size( bsize, bcount, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, aes_mode=self._aes_mode, storage_type=self._type_name, header_data=header_data), True) self.assertEqual( flen > PathORAM.compute_storage_size( bsize, bcount, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, aes_mode=self._aes_mode, storage_type=self._type_name, header_data=header_data, ignore_header=True), True) with PathORAM(fname, fsetup.stash, fsetup.position_map, key=fsetup.key, storage_type=self._type_name, **self._kwds) as f: self.assertEqual(f.header_data, header_data) self.assertEqual(fsetup.header_data, header_data) self.assertEqual(f.key, fsetup.key) self.assertEqual(f.block_size, bsize) self.assertEqual(fsetup.block_size, bsize) self.assertEqual(f.block_count, bcount) self.assertEqual(fsetup.block_count, bcount) self.assertEqual(f.storage_name, fname) self.assertEqual(fsetup.storage_name, fname) os.remove(fname)
def main(): print("Storage Name: %s" % (storage_name)) print("Block Count: %s" % (block_count)) print("Block Size: %s" % (MemorySize(block_size))) print("Total Memory: %s" % (MemorySize(block_size*block_count))) print("Actual Storage Required: %s" % (MemorySize( PathORAM.compute_storage_size( block_size, block_count, storage_type='sftp')))) print("") # Start an SSH client using paramiko print("Starting SSH Client") with paramiko.SSHClient() as ssh: ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.load_system_host_keys() ssh.connect(ssh_host, username=ssh_username, password=ssh_password) print("Setting Up Path ORAM Storage") setup_start = time.time() with PathORAM.setup(storage_name, block_size, block_count, storage_type='sftp', sshclient=ssh, ignore_existing=True) as f: print("Total Setup Time: %.2f s" % (time.time()-setup_start)) print("Current Stash Size: %s" % len(f.stash)) print("Total Data Transmission: %s" % (MemorySize(f.bytes_sent + f.bytes_received))) print("") # We close the device and reopen it after # setup to reset the bytes sent and bytes # received stats. with PathORAM(storage_name, f.stash, f.position_map, key=f.key, storage_type='sftp', sshclient=ssh) as f: test_count = 100 start_time = time.time() for t in tqdm.tqdm(list(range(test_count)), desc="Running I/O Performance Test"): f.read_block(random.randint(0,f.block_count-1)) stop_time = time.time() print("Current Stash Size: %s" % len(f.stash)) print("Access Block Avg. Data Transmitted: %s (%.3fx)" % (MemorySize((f.bytes_sent + f.bytes_received)/float(test_count)), (f.bytes_sent + f.bytes_received)/float(test_count)/float(block_size))) print("Fetch Block Avg. Latency: %.2f ms" % ((stop_time-start_time)/float(test_count)*1000)) print("")
def test_update_header_data(self): fname = ".".join(self.id().split(".")[1:]) fname += ".bin" fname = os.path.join(thisdir, fname) if os.path.exists(fname): os.remove(fname) # pragma: no cover bsize = 10 bcount = 11 header_data = bytes(bytearray([0,1,2])) fsetup = PathORAM.setup( fname, block_size=bsize, block_count=bcount, bucket_capacity=self._bucket_capacity, heap_base=self._heap_base, key=self._test_key, key_size=self._test_key_size, header_data=header_data, **self._kwds) fsetup.close() new_header_data = bytes(bytearray([1,1,1])) with PathORAM(fname, fsetup.stash, fsetup.position_map, key=fsetup.key, storage_type=self._type_name, **self._kwds) as f: self.assertEqual(f.header_data, header_data) f.update_header_data(new_header_data) self.assertEqual(f.header_data, new_header_data) with PathORAM(fname, fsetup.stash, fsetup.position_map, key=fsetup.key, storage_type=self._type_name, **self._kwds) as f: self.assertEqual(f.header_data, new_header_data) with self.assertRaises(ValueError): with PathORAM(fname, fsetup.stash, fsetup.position_map, key=fsetup.key, storage_type=self._type_name, **self._kwds) as f: f.update_header_data(bytes(bytearray([1,1]))) with self.assertRaises(ValueError): with PathORAM(fname, fsetup.stash, fsetup.position_map, key=fsetup.key, storage_type=self._type_name, **self._kwds) as f: f.update_header_data(bytes(bytearray([1,1,1,1]))) with PathORAM(fname, fsetup.stash, fsetup.position_map, key=fsetup.key, storage_type=self._type_name, **self._kwds) as f: self.assertEqual(f.header_data, new_header_data) os.remove(fname)
return ''.join(random.choice(letters) for i in range(stringLength)) if __name__ == "__main__": ap = ArgumentParser(description="runs the PrivateKV bucket load test") ap.add_argument("-z", default=4, type=int, help="number of blocks in a bucket") args = ap.parse_args() print("Setting up ORAM") oram = PathORAM.setup("test", BLOCK_SIZE, BLOCK_COUNT, storage_type="ram", ignore_existing=True, cached_levels=0, bucket_capacity=args.z) kv = privatekv.store.KVORAM(oram) n = 1000000 for i in range(n): key = random.randint(0, 100000) kv.put(key, randomString()) if 0 == i % 1000: print("inserts %d / %d" % (i, n)) print("computing level loads") loads = privatekv.utils.compute_avg_level_load(oram)
def test_locked_flag(self): with PathORAM(self._testfname, self._stash, self._position_map, key=self._key, storage_type=self._type_name, **self._kwds) as f: with self.assertRaises(IOError): with PathORAM(self._testfname, self._stash, self._position_map, key=self._key, storage_type=self._type_name, **self._kwds) as f1: pass # pragma: no cover with self.assertRaises(IOError): with PathORAM(self._testfname, self._stash, self._position_map, key=self._key, storage_type=self._type_name, **self._kwds) as f1: pass # pragma: no cover with PathORAM(self._testfname, self._stash, self._position_map, key=self._key, storage_type=self._type_name, ignore_lock=True, **self._kwds) as f1: pass with self.assertRaises(IOError): with PathORAM(self._testfname, self._stash, self._position_map, key=self._key, storage_type=self._type_name, **self._kwds) as f1: pass # pragma: no cover with PathORAM(self._testfname, self._stash, self._position_map, key=self._key, storage_type=self._type_name, ignore_lock=True, **self._kwds) as f1: pass with PathORAM(self._testfname, self._stash, self._position_map, key=self._key, storage_type=self._type_name, ignore_lock=True, **self._kwds) as f1: pass with PathORAM(self._testfname, self._stash, self._position_map, key=self._key, storage_type=self._type_name, **self._kwds) as f: pass