def test_update_overwrite_file_in_directory():
test_attributes = {
"name": "CLARK_KENT.TXT",
"code_name": codename1,
"size": 34,
"key": generate_key()
}
replacement_attributes = {
"name": "CLARK_KENT.TXT",
"code_name": codename2,
"size": 23,
"key": generate_key()
}
target_location = os.path.join("dir", test_attributes["name"])
expected_node = File.from_values(**replacement_attributes)
manifest = Manifest()
manifest.update_file(target_location, test_attributes["code_name"],
test_attributes["size"], test_attributes["key"])
manifest.update_file(target_location, replacement_attributes["code_name"],
replacement_attributes["size"],
replacement_attributes["key"])
get_results = manifest.get(target_location)
assert get_results == expected_node
def test_update_bad_path():
manifest = Manifest()
manifest.update_file("dir", codename1, 34, generate_key())
with pytest.raises(exceptions.InvalidPath):
manifest.update_file(os.path.join("dir", "CLARK_KENT.TXT"), codename2,
53, generate_key())
def test_update_overwrite_file():
test_attributes = {
"name": "CLARK_KENT.TXT",
"code_name": codename1,
"size": 34,
"key": generate_key()
}
replacement_attributes = {
"name": "CLARK_KENT.TXT",
"code_name": codename2,
"size": 23,
"key": generate_key()
}
expected_old_node = File.from_values(**test_attributes)
expected_replacement_node = File.from_values(**replacement_attributes)
manifest = Manifest()
manifest.update_file(test_attributes["name"], test_attributes["code_name"],
test_attributes["size"], test_attributes["key"])
old_file = manifest.update_file(replacement_attributes["name"],
replacement_attributes["code_name"],
replacement_attributes["size"],
replacement_attributes["key"])
assert old_file == expected_old_node
get_results = manifest.get(replacement_attributes["name"])
assert get_results == expected_replacement_node
def test_move_to_existing():
manifest = Manifest()
manifest.update_file("file", codename1, 3, generate_key())
manifest.update_file("new_file", codename1, 3, generate_key())
node = manifest.get("file")
with pytest.raises(exceptions.InvalidPath):
manifest.move("file", "new_file")
def test_interesting_serialization():
manifest = Manifest()
manifest.update_file(os.path.join("dir1", "CLARK_KENT.TXT"), codename1, 34,
generate_key())
manifest.update_file(os.path.join("dir1", "dir2", "KAL_EL.xls"), codename2,
42, generate_key())
manifest.update_file("SUPERMAN.PDF", codename3, 32, generate_key())
manifest.update_file("MAN_OF_STEEL.JPG", codename4, 23, generate_key())
manifest_string = manifest.serialize()
reconstructed_manifest = Manifest.deserialize(manifest_string)
assert manifest == reconstructed_manifest
def put(self, filename, data, key=None):
"""
Args:
filename: string
data: bytestring
key: an optional key used to encrypt
Returns:
key: bytestring of the key used for encryption
Raises:
FatalOperationFailure if any provider failed
"""
# encrypt
if key is None:
key = encryption.generate_key()
ciphertext = encryption.encrypt(data, key)
# compute RS
shares = erasure_encoding.share(ciphertext,
self.file_reconstruction_threshold,
self.num_providers)
# upload to each provider
def upload(provider, share):
provider.put(filename, share)
failures = run_parallel(upload, zip(self.providers, shares))
if len(failures) > 0:
raise exceptions.FatalOperationFailure(failures)
return key
def test_move_into_self():
manifest = Manifest()
manifest.update_file("foo/file", codename1, 3, generate_key())
node = manifest.get("foo")
manifest.move("foo", "foo/foo")
assert node == manifest.get("foo/foo")
def provision(providers, bootstrap_reconstruction_threshold,
file_reconstruction_threshold):
"""
Create a new Daruma.
Warning: Deletes all files on all providers! Even if a FatalOperationFailure is thrown, files on all providers will be unstable or deleted.
Args:
providers: a list of providers
bootstrap_reconstruction_threshold: the number of providers that need to be up to recover the key. Between 1 and len(providers)-1, inclusive
file_reconstruction_threshold: the number of providers that need to be up to read files, given the key. Between 1 and len(providers)-1, inclusive
Returns a constructed Daruma object
Raises:
ValueError if arguments are invalid
FatalOperationFailure if provisioning failed
"""
logger.debug("provisioning: brt=%d, frt=%d",
bootstrap_reconstruction_threshold,
file_reconstruction_threshold)
Daruma._assert_valid_params(providers,
bootstrap_reconstruction_threshold,
file_reconstruction_threshold)
# make a copy of providers so that changes to the external list doesn't affect this one
providers = providers[:]
def wipe(provider):
provider.wipe()
failures = run_parallel(wipe,
map(lambda provider: [provider], providers))
if len(failures) > 0:
raise exceptions.FatalOperationFailure(failures)
master_key = generate_key()
manifest_name = generate_random_name()
bootstrap = Bootstrap(master_key, manifest_name,
file_reconstruction_threshold)
# the bootstrap manager uses SSSS
bootstrap_manager = BootstrapManager(
providers, bootstrap_reconstruction_threshold)
try:
bootstrap_manager.distribute_bootstrap(bootstrap)
except exceptions.FatalOperationFailure:
# TODO check for network error
raise
file_manager = FileManager(providers,
len(providers),
file_reconstruction_threshold,
master_key,
manifest_name,
setup=True)
resilience_manager = ResilienceManager(providers, file_manager,
bootstrap_manager)
return Daruma(bootstrap_manager,
file_manager,
resilience_manager,
load_manifest=False)
def test_move_file():
manifest = Manifest()
manifest.update_file("dir/file", codename1, 3, generate_key())
node = manifest.get("dir/file")
manifest.move("dir/file", "dir/new_file")
assert node == manifest.get("dir/new_file")
with pytest.raises(exceptions.InvalidPath):
manifest.get("dir/file")
def test_nested_move():
manifest = Manifest()
manifest.create_directory("bar")
manifest.update_file("foo/file", codename1, 3, generate_key())
node = manifest.get("foo")
manifest.move("foo", "bar/foo2")
assert node == manifest.get("bar/foo2")
with pytest.raises(exceptions.InvalidPath):
manifest.get("foo")
def test_mkdir_over_existing():
manifest = Manifest()
manifest.update_file(os.path.join("dir", "CLARK_KENT.TXT"), codename1, 23,
generate_key())
expected_root = manifest.get("")
manifest.create_directory("dir")
root = manifest.get("")
assert root == expected_root
def test_update_directory():
test_attributes = {
"name": "CLARK_KENT.TXT",
"code_name": codename1,
"size": 34,
"key": generate_key()
}
expected_node = File.from_values(**test_attributes)
manifest = Manifest()
target_location = os.path.join("dir", test_attributes["name"])
manifest.update_file(target_location, test_attributes["code_name"],
test_attributes["size"], test_attributes["key"])
with pytest.raises(exceptions.InvalidPath):
manifest.update_file("dir", codename2, 34, generate_key())
# Test that the manifest hasn't been modified
get_results = manifest.get(target_location)
assert get_results == expected_node
def test_interesting_manifest_equals():
key1 = generate_key()
key2 = generate_key()
key3 = generate_key()
key4 = generate_key()
manifest1 = Manifest()
manifest1.update_file(os.path.join("dir1", "CLARK_KENT.TXT"), codename1,
34, key1)
manifest1.update_file(os.path.join("dir1", "dir2", "KAL_EL.xls"),
codename2, 42, key2)
manifest1.update_file("SUPERMAN.PDF", codename3, 32, key3)
manifest1.update_file("MAN_OF_STEEL.JPG", codename4, 23, key4)
manifest2 = Manifest()
manifest2.update_file(os.path.join("dir1", "CLARK_KENT.TXT"), codename1,
34, key1)
manifest2.update_file(os.path.join("dir1", "dir2", "KAL_EL.xls"),
codename2, 42, key2)
manifest2.update_file("SUPERMAN.PDF", codename3, 32, key3)
manifest2.update_file("MAN_OF_STEEL.JPG", codename4, 23, key4)
assert manifest1 == manifest2
def test_get_file_key():
test_attributes = {
"name": "CLARK_KENT.TXT",
"code_name": codename1,
"size": 34,
"key": generate_key()
}
manifest = Manifest()
manifest.update_file(test_attributes["name"], test_attributes["code_name"],
test_attributes["size"], test_attributes["key"])
get_results = manifest.get(test_attributes["name"])
assert get_results.key == test_attributes["key"]
def test_generate_nodes_tree():
key1 = generate_key()
manifest = Manifest()
asdf_file = File.from_values("asdf", codename1, 23, key1)
expected_files = [("bar", File.from_values("bar", codename1, 23, key1)),
("dir2", Directory.from_values("dir2", [asdf_file])),
("dir2/asdf", asdf_file)]
created_filepaths = [
"foo", "asd", "dir1/bar", "dir1/dir2/asdf", "dir3/baz"
]
for path in created_filepaths:
manifest.update_file(path, codename1, 23, key1)
assert sorted(
manifest.generate_nodes_under("dir1")) == sorted(expected_files)
def test_update_new_file():
test_attributes = {
"name": "CLARK_KENT.TXT",
"code_name": codename1,
"size": 34,
"key": generate_key()
}
expected_node = File.from_values(**test_attributes)
manifest = Manifest()
old_code_name = manifest.update_file(test_attributes["name"],
test_attributes["code_name"],
test_attributes["size"],
test_attributes["key"])
assert old_code_name is None
get_results = manifest.get(test_attributes["name"])
assert get_results == expected_node
def test_remove_directory():
test_attributes = {
"name": "CLARK_KENT.TXT",
"code_name": codename1,
"size": 34,
"key": generate_key()
}
target_location = os.path.join("dir", test_attributes["name"])
manifest = Manifest()
manifest.update_file(target_location, test_attributes["code_name"],
test_attributes["size"], test_attributes["key"])
expected_node = manifest.get("dir")
remove_results = manifest.remove("dir")
assert remove_results == expected_node
result_children = manifest.get("").get_children()
assert result_children == []
def test_mkdir_subdirectory():
test_attributes = {
"name": "CLARK_KENT.TXT",
"code_name": codename1,
"size": 34,
"key": generate_key()
}
expected_file = File.from_values(**test_attributes)
expected_dir2 = Directory.from_values("dir2")
expected_dir = Directory.from_values("dir", [expected_file, expected_dir2])
expected_root = Directory.from_values("", [expected_dir])
manifest = Manifest()
manifest.update_file(os.path.join("dir", test_attributes["name"]),
test_attributes["code_name"], test_attributes["size"],
test_attributes["key"])
manifest.create_directory(os.path.join("dir", "dir2"))
root = manifest.get("")
assert root == expected_root
def test_remove_file():
test_attributes = {
"name": "CLARK_KENT.TXT",
"code_name": codename1,
"size": 34,
"key": generate_key()
}
target_location = os.path.join("dir", test_attributes["name"])
expected_node = File.from_values(**test_attributes)
manifest = Manifest()
manifest.update_file(target_location, test_attributes["code_name"],
test_attributes["size"], test_attributes["key"])
remove_results = manifest.remove(target_location)
assert remove_results == expected_node
result_children = manifest.get("dir").get_children()
assert result_children == []
with pytest.raises(exceptions.InvalidPath):
manifest.get(target_location)
def test_roundtrip_with_key():
key = generate_key()
FD = FileDistributor(providers, len(providers), 3)
FD.put("test", "data", key)
assert FD.get("test", key) == "data"
def test_wrong_key():
wrong_key = generate_key()
FD = FileDistributor(providers, len(providers), 3)
FD.put("test", "data")
with pytest.raises(exceptions.FatalOperationFailure):
FD.get("test", wrong_key)
def test_update_root():
manifest = Manifest()
with pytest.raises(exceptions.InvalidPath):
manifest.update_file("", codename1, 32, generate_key())
def test_mkdir_with_conflicting_file():
manifest = Manifest()
manifest.update_file("dir", codename1, 23, generate_key())
with pytest.raises(exceptions.InvalidPath):
manifest.create_directory("dir")
def test_remove_invalid_parent_is_file():
manifest = Manifest()
manifest.update_file("CLARK_KENT", codename1, 34, generate_key())
with pytest.raises(exceptions.InvalidPath):
manifest.remove(os.path.join("CLARK_KENT", "asfasdfa"))
