def test_block_crud(self):
driver = self.create_driver()
block_size = 3000
vault_id = self.create_vault_id()
driver.create_vault(vault_id)
# Create a file-like object
block_data = MockFile(block_size)
# Test Invalid block_id, ie, wrong sha1 hash.
storage_id = ""
try:
status, storage_id = driver.store_block(vault_id,
"test_disk_trouble_file",
os.urandom(10))
except:
assert True
driver.delete_block(vault_id, storage_id)
assert (driver.get_block_object_length(vault_id,
storage_id) == 0)
# Test delete invalid block
driver.delete_block(vault_id, "test_invalid_block_deletion")
# Test valid block_id.
block_id = block_data.sha1()
status, storage_id = driver.store_block(
vault_id, block_id, block_data.read())
block_data.seek(0)
assert driver.block_exists(vault_id, storage_id)
# Read back the block data and compare
file_obj = driver.get_block_obj(vault_id, storage_id)
returned_data = file_obj.read()
# Returned data should be exatly the same
assert len(returned_data) == block_size
assert returned_data == block_data._content
assert (driver.get_block_object_length(vault_id, storage_id)
== block_size)
driver.delete_block(vault_id, storage_id)
assert not driver.block_exists(vault_id, storage_id)
assert None == driver.get_block_obj(vault_id, 'invalid_block_id')
assert driver.delete_vault(vault_id)
def test_multi_block_crud(self):
driver = self.create_driver()
block_size = 3000
vault_id = 'multi_block_crud_vault_test'
projectid = 'multi_block_test_project_id'
driver.create_vault(vault_id)
block_datas = [MockFile(block_size) for _ in range(3)]
block_ids = [block_data.sha1() for block_data in block_datas]
(status, storage_ids) = driver.store_async_block(
vault_id, block_ids,
[block_data.read() for block_data in block_datas])
for storage_id, block_data in zip(storage_ids, block_datas):
assert driver.block_exists(vault_id, storage_id)
# Read back the block data and compare
file_obj = driver.get_block_obj(vault_id, storage_id)
returned_data = file_obj.read()
# Returned data should be exatly the same
assert len(returned_data) == block_size
assert returned_data == block_data._content
driver.delete_block(vault_id, storage_id)
assert not driver.block_exists(vault_id, storage_id)
assert None == driver.get_block_obj(vault_id, 'invalid_block_id')
assert driver.delete_vault(vault_id)
def test_block_generator(self):
driver = self.create_driver()
block_size = 3000
vault_id = self.create_vault_id()
driver.create_vault(vault_id)
# Test re-entrance
driver.create_vault(vault_id)
blocks = [MockFile(block_size) for x in range(0, 10)]
orig_hash = md5()
for block_data in blocks:
orig_hash.update(block_data._content)
orig_hex = orig_hash.hexdigest()
block_ids = []
storage_ids = []
for block_data in blocks:
block_id = block_data.sha1()
block_ids.append(block_id)
status, storage_id = driver.store_block(vault_id, block_id,
block_data.read())
storage_ids.append(storage_id)
block_data.seek(0)
# Now call the block generator.
blockid_gen = storage_ids[:]
gen = driver.create_blocks_generator(vault_id, blockid_gen)
fetched_data = list(gen)
assert len(fetched_data) == len(blocks) == 10
for x in range(0, len(fetched_data)):
blocks[x].seek(0)
storage_id, obj = fetched_data[x]
self.assertEqual(obj.read(), blocks[x].read())
# Clenaup.
for storage_id in storage_ids[:]:
driver.delete_block(vault_id, storage_id)
assert driver.delete_vault(vault_id)
def test_block_crud(self):
d = DiskStorageDriver()
block_size = 3000
vault_id = 'block_crud_vault_test'
block_id = 'blah'
projectid = 'test_project_id'
d.create_vault(projectid, vault_id)
# Create a file-like object
block_data = MockFile(block_size)
d.store_block(projectid, vault_id, block_id, block_data.read())
block_data.seek(0)
assert d.block_exists(projectid, vault_id, block_id)
# Read back the block data and compare
file_obj = d.get_block_obj(projectid, vault_id, block_id)
returned_data = file_obj.read()
# Returned data should be exatly the same
assert len(returned_data) == block_size
assert returned_data == block_data._content
d.delete_block(projectid, vault_id, block_id)
d.delete_block(projectid, vault_id, block_id)
assert not d.block_exists(projectid, vault_id, block_id)
assert None == d.get_block_obj(projectid, vault_id, 'invalid_block_id')
def test_block_crud(self):
driver = self.create_driver()
block_size = 3000
vault_id = self.create_vault_id()
driver.create_vault(vault_id)
# Create a file-like object
block_data = MockFile(block_size)
# Test Invalid block_id, ie, wrong sha1 hash.
storage_id = ""
try:
status, storage_id = driver.store_block(vault_id,
"test_disk_trouble_file",
os.urandom(10))
except:
assert True
driver.delete_block(vault_id, storage_id)
assert (driver.get_block_object_length(vault_id, storage_id) == 0)
# Test delete invalid block
driver.delete_block(vault_id, "test_invalid_block_deletion")
# Test valid block_id.
block_id = block_data.sha1()
status, storage_id = driver.store_block(vault_id, block_id,
block_data.read())
block_data.seek(0)
assert driver.block_exists(vault_id, storage_id)
# Read back the block data and compare
file_obj = driver.get_block_obj(vault_id, storage_id)
returned_data = file_obj.read()
# Returned data should be exatly the same
assert len(returned_data) == block_size
assert returned_data == block_data._content
assert (driver.get_block_object_length(vault_id,
storage_id) == block_size)
driver.delete_block(vault_id, storage_id)
assert not driver.block_exists(vault_id, storage_id)
assert None == driver.get_block_obj(vault_id, 'invalid_block_id')
assert driver.delete_vault(vault_id)
def test_small_read(self):
num_files = 7
min_file_size = 0
max_file_size = 10000
file_sizes = [
randrange(min_file_size, max_file_size)
for i in range(0, num_files)
]
files = [MockFile(size) for size in file_sizes]
# Calculate an md5 of all of our files.
z = md5()
for f in files:
z.update(f._content)
expected_size = sum(file_sizes)
expected_md5 = z.hexdigest()
# FileCat only takes generators
fc = FileCat((f for f in files))
z = md5()
bytes_read = 0
while True:
buff = fc.read(99)
assert len(buff) <= 99
if len(buff) == 0:
break # DONE
bytes_read += len(buff)
z.update(buff)
computed_md5 = z.hexdigest()
assert bytes_read == sum(file_sizes)
assert computed_md5 == expected_md5
def test_vault_block_list(self):
driver = self.create_driver()
block_size = 100
vault_id = self.create_vault_id()
driver.create_vault(vault_id)
block_datas = [MockFile(block_size) for _ in range(30)]
block_ids = [block_data.sha1() for block_data in block_datas]
(status, storage_ids) = driver.store_async_block(
vault_id, block_ids,
[block_data.read() for block_data in block_datas])
for storage_id, block_data in zip(storage_ids, block_datas):
assert driver.block_exists(vault_id, storage_id)
ret_blocks = driver.get_vault_block_list(vault_id, limit=2)
self.assertEqual(len(ret_blocks), 2)
ret_blocks = driver.get_vault_block_list(self.create_vault_id(),
limit=2)
self.assertIsNone(ret_blocks)
def test_full_read(self):
num_files = 9
min_file_size = 1
max_file_size = 5
file_sizes = [
randrange(min_file_size, max_file_size)
for i in range(0, num_files)
]
files = [MockFile(size) for size in file_sizes]
# Calculate an md5 of all of our files.
z = md5()
for f in files:
z.update(f._content)
expected_size = sum(file_sizes)
expected_md5 = z.hexdigest()
# Pass None to FileCat
fc = FileCat(None)
# Pass empty list to FileCat
fc = FileCat((f for f in range(1, 0)))
# FileCat only takes generators
fc = FileCat((f for f in files))
data = fc.read() # read it all
z = md5()
z.update(data)
computed_md5 = z.hexdigest()
assert len(data) == sum(file_sizes)
assert computed_md5 == expected_md5
def create_storage_block_id(self):
return '{0:}_{1:}'.format(
MockFile(random.randrange(100)).sha1(), str(uuid.uuid4()))