def test_delete_blocks_with_references(self, finalize_status):
# Create two files each consisting of 3 blocks of size 100 bytes
file_ids = []
for _ in range(2):
response = self.simulate_post(self.get_files_path(self.vault_name),
headers=self._hdrs)
rel_url, querystring = relative_uri(
self.srmock.headers_dict['Location'])
file_ids.append(rel_url)
block_list = self.helper_create_blocks(3, singleblocksize=True)
offsets = [x * 100 for x in range(3)]
data = list(zip(block_list, offsets))
hdrs = {'content-type': 'application/x-deuce-block-list'}
hdrs.update(self._hdrs)
for file_id in file_ids:
# assign blocks to file
response = self.simulate_post(file_id + '/blocks',
body=json.dumps(data),
headers=hdrs)
if finalize_status:
# finalize file
filelength = {'x-file-length': '300'}
hdrs.update(filelength)
response = self.simulate_post(file_id, headers=hdrs)
for block in block_list:
response = self.simulate_delete(self.get_block_path(
self.vault_name, block),
headers=self._hdrs)
self.assertEqual(self.srmock.status, falcon.HTTP_409)
def test_delete_blocks_with_references(self, finalize_status):
# Create two files each consisting of 3 blocks of size 100 bytes
file_ids = []
for _ in range(2):
response = self.simulate_post(self.get_files_path(self.vault_name),
headers=self._hdrs)
rel_url, querystring = relative_uri(
self.srmock.headers_dict['Location'])
file_ids.append(rel_url)
block_list = self.helper_create_blocks(3, singleblocksize=True)
offsets = [x * 100 for x in range(3)]
data = list(zip(block_list, offsets))
hdrs = {'content-type': 'application/x-deuce-block-list'}
hdrs.update(self._hdrs)
for file_id in file_ids:
# assign blocks to file
response = self.simulate_post(file_id + '/blocks',
body=json.dumps(data), headers=hdrs)
if finalize_status:
# finalize file
filelength = {'x-file-length': '300'}
hdrs.update(filelength)
response = self.simulate_post(file_id, headers=hdrs)
for block in block_list:
response = self.simulate_delete(
self.get_block_path(self.vault_name, block),
headers=self._hdrs)
self.assertEqual(self.srmock.status, falcon.HTTP_409)
def helper_get_blocks(self,
path,
marker,
limit,
assert_ret_url,
assert_data_len,
repeat=False,
exam_block_data=False):
resp_block_list = []
params = dict()
if limit != 0:
params['limit'] = limit
if marker != 0:
params['marker'] = marker
while True:
response = self.simulate_get(path,
query_string=set_qs(path, params),
headers=self._hdrs)
next_batch = self.srmock.headers_dict.get("X-Next-Batch")
if next_batch:
next_batch_url, query_string = relative_uri(next_batch)
else:
next_batch_url = next_batch
resp_block_list += json.loads(response[0].decode())
self.assertIsInstance(json.loads(response[0].decode()), list)
if not repeat:
self.assertEqual(not next_batch_url, not assert_ret_url)
self.assertEqual(len(resp_block_list), assert_data_len)
for h in resp_block_list:
self.assertIn(h, self.block_list)
if assert_data_len == -1 or \
assert_data_len == self.total_block_num:
for h in self.block_list:
self.assertIn(h, resp_block_list)
if exam_block_data:
self.helper_exam_block_data(resp_block_list)
return next_batch
if not next_batch_url:
break
# TODO (TheSriram): Make finding marker more elegant
for query in query_string.split('&'):
if 'marker' in query:
current_marker = query.split('marker=')[1]
params['marker'] = current_marker
self.assertEqual(len(resp_block_list), assert_data_len)
for h in resp_block_list:
self.assertIn(h, self.block_list)
for h in self.block_list:
self.assertIn(h, resp_block_list)
# By default exam blocks if fetching all blocks
self.helper_exam_block_data(resp_block_list)
def test_list_blocks_with_limit_marker(self):
# Test with bad marker
block_storage_path = self.get_storage_blocks_path(self.vault_name)
block_marker = self.create_storage_block_id()
marker = 'marker={0:}'.format(block_marker)
response = self.simulate_get(self._block_storage_path,
query_string=marker,
headers=self._hdrs)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
self.assertEqual(response[0].decode(), '[]')
block_list, block_data = self.helper_create_blocks(num_blocks=40)
storage_list = self.helper_store_blocks(self.vault_name, block_data)
# Test with no marker
response = self.simulate_get(self._block_storage_path,
headers=self._hdrs)
storage_ids = json.loads(response[0].decode())
self.assertEqual(self.srmock.status, falcon.HTTP_200)
# Test valid marker
# Lets list from the second storage_id
query_marker = storage_ids[0]
marker = 'marker={0:}'.format(query_marker)
response = self.simulate_get(self._block_storage_path,
query_string=marker,
headers=self._hdrs)
responses = json.loads(response[0].decode())
for resp in responses:
resp_sha1, resp_uuid = resp.split('_')
self.assertTrue(uuid.UUID(resp_uuid))
self.assertIn(resp, storage_ids)
# Test valid limit
response = self.simulate_get(self._block_storage_path,
query_string='limit=3',
headers=self._hdrs)
self.assertEqual(len(json.loads(response[0].decode())), 3)
# Test valid marker with limit
query_string = 'marker={0:}&limit={1:}'.format(query_marker, '2')
response = self.simulate_get(self._block_storage_path,
query_string=query_string,
headers=self._hdrs)
self.assertEqual(len(json.loads(response[0].decode())), 2)
next_url, querystring = relative_uri(
self.srmock.headers_dict['x-next-batch'])
response = self.simulate_get(next_url,
query_string=querystring,
headers=self._hdrs)
self.assertEqual(len(json.loads(response[0].decode())), 2)
def test_list_blocks_with_limit_marker(self):
# Test with bad marker
block_storage_path = self.get_storage_blocks_path(self.vault_name)
block_marker = self.create_storage_block_id()
marker = 'marker={0:}'.format(block_marker)
response = self.simulate_get(self._block_storage_path,
query_string=marker,
headers=self._hdrs)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
self.assertEqual(response[0].decode(), '[]')
block_list, block_data = self.helper_create_blocks(num_blocks=40)
storage_list = self.helper_store_blocks(self.vault_name, block_data)
# Test with no marker
response = self.simulate_get(self._block_storage_path,
headers=self._hdrs)
storage_ids = json.loads(response[0].decode())
self.assertEqual(self.srmock.status, falcon.HTTP_200)
# Test valid marker
# Lets list from the second storage_id
query_marker = storage_ids[0]
marker = 'marker={0:}'.format(query_marker)
response = self.simulate_get(self._block_storage_path,
query_string=marker,
headers=self._hdrs)
responses = json.loads(response[0].decode())
for resp in responses:
resp_sha1, resp_uuid = resp.split('_')
self.assertTrue(uuid.UUID(resp_uuid))
self.assertIn(resp, storage_ids)
# Test valid limit
response = self.simulate_get(self._block_storage_path,
query_string='limit=3',
headers=self._hdrs)
self.assertEqual(len(json.loads(response[0].decode())), 3)
# Test valid marker with limit
query_string = 'marker={0:}&limit={1:}'.format(query_marker, '2')
response = self.simulate_get(self._block_storage_path,
query_string=query_string,
headers=self._hdrs)
self.assertEqual(len(json.loads(response[0].decode())), 2)
next_url, querystring = relative_uri(
self.srmock.headers_dict['x-next-batch'])
response = self.simulate_get(next_url,
query_string=querystring,
headers=self._hdrs)
self.assertEqual(len(json.loads(response[0].decode())), 2)
def helper_create_files(self, num):
hdrs = self._hdrs.copy()
hdrs['x-file-length'] = '0'
for cnt in range(0, num):
response = self.simulate_post(self._files_path, headers=self._hdrs)
file_path = self.srmock.headers_dict['Location']
file_uri, querystring = relative_uri(file_path)
response = self.simulate_post(file_uri, headers=hdrs)
file_id = urlparse(file_path).path.split('/')[-1]
self.file_list.append(file_id)
return num
def helper_get_blocks(self, path, marker, limit, assert_ret_url,
assert_data_len, repeat=False,
exam_block_data=False):
resp_block_list = []
params = dict()
if limit != 0:
params['limit'] = limit
if marker != 0:
params['marker'] = marker
while True:
response = self.simulate_get(path,
query_string=set_qs(path, params),
headers=self._hdrs)
next_batch = self.srmock.headers_dict.get("X-Next-Batch")
if next_batch:
next_batch_url, query_string = relative_uri(next_batch)
else:
next_batch_url = next_batch
resp_block_list += json.loads(response[0].decode())
self.assertIsInstance(json.loads(response[0].decode()), list)
if not repeat:
self.assertEqual(not next_batch_url, not assert_ret_url)
self.assertEqual(len(resp_block_list), assert_data_len)
for h in resp_block_list:
self.assertIn(h, self.block_list)
if assert_data_len == -1 or \
assert_data_len == self.total_block_num:
for h in self.block_list:
self.assertIn(h, resp_block_list)
if exam_block_data:
self.helper_exam_block_data(resp_block_list)
return next_batch
if not next_batch_url:
break
# TODO (TheSriram): Make finding marker more elegant
for query in query_string.split('&'):
if 'marker' in query:
current_marker = query.split('marker=')[1]
params['marker'] = current_marker
self.assertEqual(len(resp_block_list), assert_data_len)
for h in resp_block_list:
self.assertIn(h, self.block_list)
for h in self.block_list:
self.assertIn(h, resp_block_list)
# By default exam blocks if fetching all blocks
self.helper_exam_block_data(resp_block_list)
def helper_create_files(self, num):
hdrs = self._hdrs.copy()
hdrs['x-file-length'] = '0'
for cnt in range(0, num):
response = self.simulate_post(self._files_path, headers=self._hdrs)
file_path = self.srmock.headers_dict['Location']
file_uri, querystring = relative_uri(file_path)
response = self.simulate_post(file_uri, headers=hdrs)
file_id = urlparse(file_path).path.split('/')[-1]
self.file_list.append(file_id)
return num
def helper_get_files(self, marker, limit, assert_return_url,
assert_data_len, repeat=False):
resp_file_list = []
params = dict()
if limit is not None:
params['limit'] = limit
while True:
response = self.simulate_get(self._files_path,
query_string=set_qs(
self._files_path,
params),
headers=self._hdrs)
next_batch = self.srmock.headers_dict.get("X-Next-Batch")
if next_batch:
next_batch_url, query_string = relative_uri(next_batch)
else:
next_batch_url = next_batch
resp_file_list += json.loads(response[0].decode())
assert isinstance(json.loads(response[0].decode()), list)
if not repeat:
assert (not next_batch_url) == (not assert_return_url)
assert len(resp_file_list) == assert_data_len
for h in resp_file_list:
assert h in self.file_list
if assert_data_len == -1 or \
assert_data_len == self.total_file_num:
for h in self.file_list:
assert h in resp_file_list
return next_batch_url
if not next_batch_url:
break
# TODO (TheSriram): Make finding marker more elegant
for query in query_string.split('&'):
if 'marker' in query:
current_marker = query.split('marker=')[1]
params['marker'] = current_marker
self.assertEqual(len(resp_file_list), assert_data_len)
for h in resp_file_list:
assert h in self.file_list
for h in self.file_list:
assert h in resp_file_list
def helper_get_files(self, marker, limit, assert_return_url,
assert_data_len, repeat=False):
resp_file_list = []
params = dict()
if limit is not None:
params['limit'] = limit
while True:
response = self.simulate_get(self._files_path,
query_string=set_qs(
self._files_path,
params),
headers=self._hdrs)
next_batch = self.srmock.headers_dict.get("X-Next-Batch")
if next_batch:
next_batch_url, query_string = relative_uri(next_batch)
else:
next_batch_url = next_batch
resp_file_list += json.loads(response[0].decode())
assert isinstance(json.loads(response[0].decode()), list)
if not repeat:
assert (not next_batch_url) == (not assert_return_url)
assert len(resp_file_list) == assert_data_len
for h in resp_file_list:
assert h in self.file_list
if assert_data_len == -1 or \
assert_data_len == self.total_file_num:
for h in self.file_list:
assert h in resp_file_list
return next_batch_url
if not next_batch_url:
break
# TODO (TheSriram): Make finding marker more elegant
for query in query_string.split('&'):
if 'marker' in query:
current_marker = query.split('marker=')[1]
params['marker'] = current_marker
self.assertEqual(len(resp_file_list), assert_data_len)
for h in resp_file_list:
assert h in self.file_list
for h in self.file_list:
assert h in resp_file_list
def test_vault_leaf(self):
hdrs = self._hdrs
vault_path = 'http://localhost/v1.0/vaults/'
# Create an empty root path in the storage.
self.helper_create_vault('vault_0', hdrs)
self.helper_delete_vault('vault_0', hdrs)
response = self.simulate_get('/v1.0/vaults/',
headers=hdrs)
self.assertEqual(str(response[0].decode()), str('{}'))
# Prepare several vaults in the storage.
for cnt in range(5):
self.helper_create_vault('vault_{0}'.format(cnt), hdrs)
# No limit nor marker
response = self.simulate_get('/v1.0/vaults/',
headers=hdrs)
self.assertEqual(json.loads(response[0].decode()),
{"vault_3": {"url": vault_path + "vault_3"},
"vault_4": {"url": vault_path + "vault_4"},
"vault_2": {"url": vault_path + "vault_2"},
"vault_1": {"url": vault_path + "vault_1"},
"vault_0": {"url": vault_path + "vault_0"}}
)
response = self.simulate_get('/v1.0/vaults/',
query_string='marker=vault_0',
headers=hdrs)
self.assertEqual(json.loads(response[0].decode()),
{"vault_4": {"url": vault_path + "vault_4"},
"vault_2": {"url": vault_path + "vault_2"},
"vault_3": {"url": vault_path + "vault_3"},
"vault_0": {"url": vault_path + "vault_0"},
"vault_1": {"url": vault_path + "vault_1"}}
)
# Only limit
response = self.simulate_get('/v1.0/vaults/',
query_string='limit=99',
headers=hdrs)
self.assertEqual(json.loads(response[0].decode()),
{"vault_4": {"url": vault_path + "vault_4"},
"vault_2": {"url": vault_path + "vault_2"},
"vault_3": {"url": vault_path + "vault_3"},
"vault_0": {"url": vault_path + "vault_0"},
"vault_1": {"url": vault_path + "vault_1"}}
)
response = self.simulate_get('/v1.0/vaults/',
query_string='limit=1',
headers=hdrs)
self.assertEqual(json.loads(response[0].decode()),
{"vault_0": {"url": vault_path + "vault_0"}}
)
next_url = self.srmock.headers_dict["X-Next-Batch"]
uri, querystring = relative_uri(next_url)
new_querystring = querystring.replace('limit=1', 'limit=99')
response = self.simulate_get(uri,
query_string=new_querystring,
headers=hdrs)
self.assertEqual(json.loads(response[0].decode()),
{"vault_4": {"url": vault_path + "vault_4"},
"vault_2": {"url": vault_path + "vault_2"},
"vault_3": {"url": vault_path + "vault_3"},
"vault_1": {"url": vault_path + "vault_1"}}
)
response = self.simulate_get(uri,
query_string=querystring,
headers=hdrs)
self.assertEqual(json.loads(response[0].decode()),
{"vault_1": {"url": vault_path + "vault_1"}}
)
response = self.simulate_get('/v1.0/vaults/',
query_string='marker=vault_not_exists'
'&limit=99',
headers=hdrs)
self.assertEqual(str(response[0].decode()), str('{}'))
# Cleanup
for cnt in range(5):
self.helper_delete_vault('vault_{0}'.format(cnt), hdrs)
def helper_file_blocks_controller(self, file_id, hdrs):
# Get blocks of a file.
response = self.simulate_get(file_id + '/blocks', headers=hdrs)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
next_batch = self.srmock.headers_dict.get("X-Next-Batch")
if next_batch:
next_batch_url, query_string = relative_uri(next_batch)
else:
next_batch_url = next_batch
# vault does not exists
response = self.simulate_get(self._NOT_EXIST_files_path +
'/%s/blocks' % (file_id,),
headers=self._hdrs)
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# fileid does not exists
nonexistent_fileid = self.create_file_id()
response = self.simulate_get(self._files_path +
'/%s/blocks' % (nonexistent_fileid),
headers=self._hdrs)
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# Get blocks of a file. with limit not zero
response = self.simulate_get(file_id + '/blocks',
query_string='limit=5', headers=hdrs)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
next_batch = self.srmock.headers_dict.get("X-Next-Batch")
if next_batch:
next_batch_url, query_string = relative_uri(next_batch)
else:
next_batch_url = next_batch
resp_block_list = []
params = {'marker': 0}
while True:
response = self.simulate_get(file_id + '/blocks',
query_string=set_qs(file_id, params),
headers=self._hdrs)
next_batch = self.srmock.headers_dict.get("X-Next-Batch")
if next_batch:
next_batch_url, query_string = relative_uri(next_batch)
else:
next_batch_url = next_batch
resp_block_list += json.loads(response[0].decode())
assert isinstance(json.loads(response[0].decode()), list)
if not next_batch_url:
break
# TODO (TheSriram): Make finding marker more elegant
for query in query_string.split('&'):
if 'marker' in query:
current_marker = query.split('marker=')[1]
params['marker'] = current_marker
self.assertEqual(len(resp_block_list), 1.2 *
conf.api_configuration.default_returned_num)
def test_vault_leaf(self):
hdrs = self._hdrs
vault_path = 'http://localhost/v1.0/vaults/'
# Create an empty root path in the storage.
self.helper_create_vault('vault_0', hdrs)
self.helper_delete_vault('vault_0', hdrs)
response = self.simulate_get('/v1.0/vaults/', headers=hdrs)
self.assertEqual(str(response[0].decode()), str('{}'))
# Prepare several vaults in the storage.
for cnt in range(5):
self.helper_create_vault('vault_{0}'.format(cnt), hdrs)
# No limit nor marker
response = self.simulate_get('/v1.0/vaults/', headers=hdrs)
self.assertEqual(
json.loads(response[0].decode()), {
"vault_3": {
"url": vault_path + "vault_3"
},
"vault_4": {
"url": vault_path + "vault_4"
},
"vault_2": {
"url": vault_path + "vault_2"
},
"vault_1": {
"url": vault_path + "vault_1"
},
"vault_0": {
"url": vault_path + "vault_0"
}
})
response = self.simulate_get('/v1.0/vaults/',
query_string='marker=vault_0',
headers=hdrs)
self.assertEqual(
json.loads(response[0].decode()), {
"vault_4": {
"url": vault_path + "vault_4"
},
"vault_2": {
"url": vault_path + "vault_2"
},
"vault_3": {
"url": vault_path + "vault_3"
},
"vault_0": {
"url": vault_path + "vault_0"
},
"vault_1": {
"url": vault_path + "vault_1"
}
})
# Only limit
response = self.simulate_get('/v1.0/vaults/',
query_string='limit=99',
headers=hdrs)
self.assertEqual(
json.loads(response[0].decode()), {
"vault_4": {
"url": vault_path + "vault_4"
},
"vault_2": {
"url": vault_path + "vault_2"
},
"vault_3": {
"url": vault_path + "vault_3"
},
"vault_0": {
"url": vault_path + "vault_0"
},
"vault_1": {
"url": vault_path + "vault_1"
}
})
response = self.simulate_get('/v1.0/vaults/',
query_string='limit=1',
headers=hdrs)
self.assertEqual(json.loads(response[0].decode()),
{"vault_0": {
"url": vault_path + "vault_0"
}})
next_url = self.srmock.headers_dict["X-Next-Batch"]
uri, querystring = relative_uri(next_url)
new_querystring = querystring.replace('limit=1', 'limit=99')
response = self.simulate_get(uri,
query_string=new_querystring,
headers=hdrs)
self.assertEqual(
json.loads(response[0].decode()), {
"vault_4": {
"url": vault_path + "vault_4"
},
"vault_2": {
"url": vault_path + "vault_2"
},
"vault_3": {
"url": vault_path + "vault_3"
},
"vault_1": {
"url": vault_path + "vault_1"
}
})
response = self.simulate_get(uri,
query_string=querystring,
headers=hdrs)
self.assertEqual(json.loads(response[0].decode()),
{"vault_1": {
"url": vault_path + "vault_1"
}})
response = self.simulate_get('/v1.0/vaults/',
query_string='marker=vault_not_exists'
'&limit=99',
headers=hdrs)
self.assertEqual(str(response[0].decode()), str('{}'))
# Cleanup
for cnt in range(5):
self.helper_delete_vault('vault_{0}'.format(cnt), hdrs)
def helper_file_blocks_controller(self, file_id, hdrs):
# Get blocks of a file.
response = self.simulate_get(file_id + '/blocks', headers=hdrs)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
next_batch = self.srmock.headers_dict.get("X-Next-Batch")
if next_batch:
next_batch_url, query_string = relative_uri(next_batch)
else:
next_batch_url = next_batch
# vault does not exists
response = self.simulate_get(self._NOT_EXIST_files_path +
'/%s/blocks' % (file_id,),
headers=self._hdrs)
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# fileid does not exists
nonexistent_fileid = self.create_file_id()
response = self.simulate_get(self._files_path +
'/%s/blocks' % (nonexistent_fileid),
headers=self._hdrs)
self.assertEqual(self.srmock.status, falcon.HTTP_404)
# Get blocks of a file. with limit not zero
response = self.simulate_get(file_id + '/blocks',
query_string='limit=5', headers=hdrs)
self.assertEqual(self.srmock.status, falcon.HTTP_200)
next_batch = self.srmock.headers_dict.get("X-Next-Batch")
if next_batch:
next_batch_url, query_string = relative_uri(next_batch)
else:
next_batch_url = next_batch
resp_block_list = []
params = {'marker': 0}
while True:
response = self.simulate_get(file_id + '/blocks',
query_string=set_qs(file_id, params),
headers=self._hdrs)
next_batch = self.srmock.headers_dict.get("X-Next-Batch")
if next_batch:
next_batch_url, query_string = relative_uri(next_batch)
else:
next_batch_url = next_batch
resp_block_list += json.loads(response[0].decode())
assert isinstance(json.loads(response[0].decode()), list)
if not next_batch_url:
break
# TODO (TheSriram): Make finding marker more elegant
for query in query_string.split('&'):
if 'marker' in query:
current_marker = query.split('marker=')[1]
params['marker'] = current_marker
self.assertEqual(len(resp_block_list), 1.2 *
conf.api_configuration.default_returned_num)
