def test_update_overwrite_file_in_directory(): test_attributes = { "name": "CLARK_KENT.TXT", "code_name": codename1, "size": 34, "key": generate_key() } replacement_attributes = { "name": "CLARK_KENT.TXT", "code_name": codename2, "size": 23, "key": generate_key() } target_location = os.path.join("dir", test_attributes["name"]) expected_node = File.from_values(**replacement_attributes) manifest = Manifest() manifest.update_file(target_location, test_attributes["code_name"], test_attributes["size"], test_attributes["key"]) manifest.update_file(target_location, replacement_attributes["code_name"], replacement_attributes["size"], replacement_attributes["key"]) get_results = manifest.get(target_location) assert get_results == expected_node
def test_update_bad_path(): manifest = Manifest() manifest.update_file("dir", codename1, 34, generate_key()) with pytest.raises(exceptions.InvalidPath): manifest.update_file(os.path.join("dir", "CLARK_KENT.TXT"), codename2, 53, generate_key())
def test_update_overwrite_file(): test_attributes = { "name": "CLARK_KENT.TXT", "code_name": codename1, "size": 34, "key": generate_key() } replacement_attributes = { "name": "CLARK_KENT.TXT", "code_name": codename2, "size": 23, "key": generate_key() } expected_old_node = File.from_values(**test_attributes) expected_replacement_node = File.from_values(**replacement_attributes) manifest = Manifest() manifest.update_file(test_attributes["name"], test_attributes["code_name"], test_attributes["size"], test_attributes["key"]) old_file = manifest.update_file(replacement_attributes["name"], replacement_attributes["code_name"], replacement_attributes["size"], replacement_attributes["key"]) assert old_file == expected_old_node get_results = manifest.get(replacement_attributes["name"]) assert get_results == expected_replacement_node
def test_move_to_existing(): manifest = Manifest() manifest.update_file("file", codename1, 3, generate_key()) manifest.update_file("new_file", codename1, 3, generate_key()) node = manifest.get("file") with pytest.raises(exceptions.InvalidPath): manifest.move("file", "new_file")
def test_interesting_serialization(): manifest = Manifest() manifest.update_file(os.path.join("dir1", "CLARK_KENT.TXT"), codename1, 34, generate_key()) manifest.update_file(os.path.join("dir1", "dir2", "KAL_EL.xls"), codename2, 42, generate_key()) manifest.update_file("SUPERMAN.PDF", codename3, 32, generate_key()) manifest.update_file("MAN_OF_STEEL.JPG", codename4, 23, generate_key()) manifest_string = manifest.serialize() reconstructed_manifest = Manifest.deserialize(manifest_string) assert manifest == reconstructed_manifest
def put(self, filename, data, key=None): """ Args: filename: string data: bytestring key: an optional key used to encrypt Returns: key: bytestring of the key used for encryption Raises: FatalOperationFailure if any provider failed """ # encrypt if key is None: key = encryption.generate_key() ciphertext = encryption.encrypt(data, key) # compute RS shares = erasure_encoding.share(ciphertext, self.file_reconstruction_threshold, self.num_providers) # upload to each provider def upload(provider, share): provider.put(filename, share) failures = run_parallel(upload, zip(self.providers, shares)) if len(failures) > 0: raise exceptions.FatalOperationFailure(failures) return key
def test_move_into_self(): manifest = Manifest() manifest.update_file("foo/file", codename1, 3, generate_key()) node = manifest.get("foo") manifest.move("foo", "foo/foo") assert node == manifest.get("foo/foo")
def provision(providers, bootstrap_reconstruction_threshold, file_reconstruction_threshold): """ Create a new Daruma. Warning: Deletes all files on all providers! Even if a FatalOperationFailure is thrown, files on all providers will be unstable or deleted. Args: providers: a list of providers bootstrap_reconstruction_threshold: the number of providers that need to be up to recover the key. Between 1 and len(providers)-1, inclusive file_reconstruction_threshold: the number of providers that need to be up to read files, given the key. Between 1 and len(providers)-1, inclusive Returns a constructed Daruma object Raises: ValueError if arguments are invalid FatalOperationFailure if provisioning failed """ logger.debug("provisioning: brt=%d, frt=%d", bootstrap_reconstruction_threshold, file_reconstruction_threshold) Daruma._assert_valid_params(providers, bootstrap_reconstruction_threshold, file_reconstruction_threshold) # make a copy of providers so that changes to the external list doesn't affect this one providers = providers[:] def wipe(provider): provider.wipe() failures = run_parallel(wipe, map(lambda provider: [provider], providers)) if len(failures) > 0: raise exceptions.FatalOperationFailure(failures) master_key = generate_key() manifest_name = generate_random_name() bootstrap = Bootstrap(master_key, manifest_name, file_reconstruction_threshold) # the bootstrap manager uses SSSS bootstrap_manager = BootstrapManager( providers, bootstrap_reconstruction_threshold) try: bootstrap_manager.distribute_bootstrap(bootstrap) except exceptions.FatalOperationFailure: # TODO check for network error raise file_manager = FileManager(providers, len(providers), file_reconstruction_threshold, master_key, manifest_name, setup=True) resilience_manager = ResilienceManager(providers, file_manager, bootstrap_manager) return Daruma(bootstrap_manager, file_manager, resilience_manager, load_manifest=False)
def test_move_file(): manifest = Manifest() manifest.update_file("dir/file", codename1, 3, generate_key()) node = manifest.get("dir/file") manifest.move("dir/file", "dir/new_file") assert node == manifest.get("dir/new_file") with pytest.raises(exceptions.InvalidPath): manifest.get("dir/file")
def test_nested_move(): manifest = Manifest() manifest.create_directory("bar") manifest.update_file("foo/file", codename1, 3, generate_key()) node = manifest.get("foo") manifest.move("foo", "bar/foo2") assert node == manifest.get("bar/foo2") with pytest.raises(exceptions.InvalidPath): manifest.get("foo")
def test_mkdir_over_existing(): manifest = Manifest() manifest.update_file(os.path.join("dir", "CLARK_KENT.TXT"), codename1, 23, generate_key()) expected_root = manifest.get("") manifest.create_directory("dir") root = manifest.get("") assert root == expected_root
def test_update_directory(): test_attributes = { "name": "CLARK_KENT.TXT", "code_name": codename1, "size": 34, "key": generate_key() } expected_node = File.from_values(**test_attributes) manifest = Manifest() target_location = os.path.join("dir", test_attributes["name"]) manifest.update_file(target_location, test_attributes["code_name"], test_attributes["size"], test_attributes["key"]) with pytest.raises(exceptions.InvalidPath): manifest.update_file("dir", codename2, 34, generate_key()) # Test that the manifest hasn't been modified get_results = manifest.get(target_location) assert get_results == expected_node
def test_interesting_manifest_equals(): key1 = generate_key() key2 = generate_key() key3 = generate_key() key4 = generate_key() manifest1 = Manifest() manifest1.update_file(os.path.join("dir1", "CLARK_KENT.TXT"), codename1, 34, key1) manifest1.update_file(os.path.join("dir1", "dir2", "KAL_EL.xls"), codename2, 42, key2) manifest1.update_file("SUPERMAN.PDF", codename3, 32, key3) manifest1.update_file("MAN_OF_STEEL.JPG", codename4, 23, key4) manifest2 = Manifest() manifest2.update_file(os.path.join("dir1", "CLARK_KENT.TXT"), codename1, 34, key1) manifest2.update_file(os.path.join("dir1", "dir2", "KAL_EL.xls"), codename2, 42, key2) manifest2.update_file("SUPERMAN.PDF", codename3, 32, key3) manifest2.update_file("MAN_OF_STEEL.JPG", codename4, 23, key4) assert manifest1 == manifest2
def test_get_file_key(): test_attributes = { "name": "CLARK_KENT.TXT", "code_name": codename1, "size": 34, "key": generate_key() } manifest = Manifest() manifest.update_file(test_attributes["name"], test_attributes["code_name"], test_attributes["size"], test_attributes["key"]) get_results = manifest.get(test_attributes["name"]) assert get_results.key == test_attributes["key"]
def test_generate_nodes_tree(): key1 = generate_key() manifest = Manifest() asdf_file = File.from_values("asdf", codename1, 23, key1) expected_files = [("bar", File.from_values("bar", codename1, 23, key1)), ("dir2", Directory.from_values("dir2", [asdf_file])), ("dir2/asdf", asdf_file)] created_filepaths = [ "foo", "asd", "dir1/bar", "dir1/dir2/asdf", "dir3/baz" ] for path in created_filepaths: manifest.update_file(path, codename1, 23, key1) assert sorted( manifest.generate_nodes_under("dir1")) == sorted(expected_files)
def test_update_new_file(): test_attributes = { "name": "CLARK_KENT.TXT", "code_name": codename1, "size": 34, "key": generate_key() } expected_node = File.from_values(**test_attributes) manifest = Manifest() old_code_name = manifest.update_file(test_attributes["name"], test_attributes["code_name"], test_attributes["size"], test_attributes["key"]) assert old_code_name is None get_results = manifest.get(test_attributes["name"]) assert get_results == expected_node
def test_remove_directory(): test_attributes = { "name": "CLARK_KENT.TXT", "code_name": codename1, "size": 34, "key": generate_key() } target_location = os.path.join("dir", test_attributes["name"]) manifest = Manifest() manifest.update_file(target_location, test_attributes["code_name"], test_attributes["size"], test_attributes["key"]) expected_node = manifest.get("dir") remove_results = manifest.remove("dir") assert remove_results == expected_node result_children = manifest.get("").get_children() assert result_children == []
def test_mkdir_subdirectory(): test_attributes = { "name": "CLARK_KENT.TXT", "code_name": codename1, "size": 34, "key": generate_key() } expected_file = File.from_values(**test_attributes) expected_dir2 = Directory.from_values("dir2") expected_dir = Directory.from_values("dir", [expected_file, expected_dir2]) expected_root = Directory.from_values("", [expected_dir]) manifest = Manifest() manifest.update_file(os.path.join("dir", test_attributes["name"]), test_attributes["code_name"], test_attributes["size"], test_attributes["key"]) manifest.create_directory(os.path.join("dir", "dir2")) root = manifest.get("") assert root == expected_root
def test_remove_file(): test_attributes = { "name": "CLARK_KENT.TXT", "code_name": codename1, "size": 34, "key": generate_key() } target_location = os.path.join("dir", test_attributes["name"]) expected_node = File.from_values(**test_attributes) manifest = Manifest() manifest.update_file(target_location, test_attributes["code_name"], test_attributes["size"], test_attributes["key"]) remove_results = manifest.remove(target_location) assert remove_results == expected_node result_children = manifest.get("dir").get_children() assert result_children == [] with pytest.raises(exceptions.InvalidPath): manifest.get(target_location)
def test_roundtrip_with_key(): key = generate_key() FD = FileDistributor(providers, len(providers), 3) FD.put("test", "data", key) assert FD.get("test", key) == "data"
def test_wrong_key(): wrong_key = generate_key() FD = FileDistributor(providers, len(providers), 3) FD.put("test", "data") with pytest.raises(exceptions.FatalOperationFailure): FD.get("test", wrong_key)
def test_update_root(): manifest = Manifest() with pytest.raises(exceptions.InvalidPath): manifest.update_file("", codename1, 32, generate_key())
def test_mkdir_with_conflicting_file(): manifest = Manifest() manifest.update_file("dir", codename1, 23, generate_key()) with pytest.raises(exceptions.InvalidPath): manifest.create_directory("dir")
def test_remove_invalid_parent_is_file(): manifest = Manifest() manifest.update_file("CLARK_KENT", codename1, 34, generate_key()) with pytest.raises(exceptions.InvalidPath): manifest.remove(os.path.join("CLARK_KENT", "asfasdfa"))