def test_account_refresh_all(self):
# clear accounts
accounts = self.api.account_list()
for account in accounts:
try:
self.api.account_flush(account)
self.api.account_delete(account)
except exc.NoSuchAccount: # account remove in the meantime
pass
# With 0 account
self.api.account_refresh()
# With 2 accounts
account1 = random_str(32)
self.api.account_create(account1)
account2 = random_str(32)
self.api.account_create(account2)
self.api.account_refresh()
res = self.api.account_show(account1)
self.assertEqual(res["bytes"], 0)
self.assertEqual(res["objects"], 0)
self.assertEqual(res["containers"], 0)
res = self.api.account_show(account2)
self.assertEqual(res["bytes"], 0)
self.assertEqual(res["objects"], 0)
self.assertEqual(res["containers"], 0)
self.api.account_delete(account1)
self.api.account_delete(account2)
def test_download_with_missing_chunks(self):
container = random_str(8)
obj = random_str(8)
expected_data = random_data(10)
chunks, _, _, meta = self.storage.object_create_ext(self.account,
container,
obj_name=obj,
data=expected_data,
policy='EC')
storage_method = STORAGE_METHODS.load(meta['chunk_method'])
sorted_chunks = _sort_chunks(chunks, storage_method.ec)
for i in range(storage_method.ec_nb_parity):
data = b''
for pos in range(len(sorted_chunks)):
chunk = random.choice(sorted_chunks[pos])
sorted_chunks[pos].remove(chunk)
resp = self._download_metachunk(meta, sorted_chunks[pos])
self.assertEqual(200, resp.status)
data += resp.data
self.assertEqual(expected_data, data)
for pos in range(len(sorted_chunks)):
chunk = random.choice(sorted_chunks[pos])
sorted_chunks[pos].remove(chunk)
resp = self._download_metachunk(meta, sorted_chunks[pos])
self.assertEqual(500, resp.status)
def test_account_flush(self):
# account_flush on unknown account
account = random_str(32)
self.assertRaises(exc.NoSuchAccount, self.api.account_flush, account)
# account_flush on existing account
name1 = random_str(32)
self.api.container_create(account, name1)
name2 = random_str(32)
self.api.container_create(account, name2)
time.sleep(0.5) # ensure container event have been processed
self.api.account_flush(account)
containers = self.api.container_list(account)
self.assertEqual(len(containers), 0)
res = self.api.account_show(account)
self.assertEqual(res["bytes"], 0)
self.assertEqual(res["objects"], 0)
self.assertEqual(res["containers"], 0)
self.api.container_delete(account, name1)
self.api.container_delete(account, name2)
time.sleep(0.5) # ensure container event have been processed
self.api.account_delete(account)
# account_flush on deleted account
self.assertRaises(exc.NoSuchAccount, self.api.account_flush, account)
def _put_chunk(self):
account = random_str(16)
container = random_str(16)
cid = cid_from_name(account, container)
content_path = random_str(16)
content_version = 1234567890
content_id = random_id(32)
fullpath = encode_fullpath(account, container, content_path,
content_version, content_id)
chunk_id = random_chunk_id()
data = random_buffer(string.printable, 100)
meta = {
'full_path': fullpath,
'container_id': cid,
'content_path': content_path,
'version': content_version,
'id': content_id,
'chunk_method': 'ec/algo=liberasurecode_rs_vand,k=6,m=3',
'policy': 'TESTPOLICY',
'chunk_hash': md5(data).hexdigest().upper(),
'oio_version': OIO_VERSION,
'chunk_pos': 0,
'metachunk_hash': md5().hexdigest(),
'metachunk_size': 1024
}
self.blob_client.chunk_put('http://' + self.rawx_id + '/' + chunk_id,
meta, data)
sleep(1) # ensure chunk event have been processed
return account, container, cid, content_path, content_version, \
content_id, chunk_id
def test_object_create_perfdata(self):
perfdata = dict()
container = random_str(8)
obj = random_str(8)
self.api.object_create(self.account,
container,
obj_name=obj,
data=obj,
perfdata=perfdata)
meta, chunks = self.api.object_locate(self.account, container, obj)
self.assertIn('proxy', perfdata)
self.assertIn('resolve', perfdata['proxy'])
self.assertIn('meta2', perfdata['proxy'])
self.assertIn('overall', perfdata['proxy'])
self.assertIn('rawx', perfdata)
if meta['policy'] == 'EC':
self.assertIn('ec', perfdata['rawx'])
for chunk in chunks:
self.assertIn('connect.' + chunk['url'], perfdata['rawx'])
self.assertIn('upload.' + chunk['url'], perfdata['rawx'])
self.assertIn('connect.AVG', perfdata['rawx'])
self.assertIn('connect.SD', perfdata['rawx'])
self.assertIn('connect.RSD', perfdata['rawx'])
self.assertIn('upload.AVG', perfdata['rawx'])
self.assertIn('upload.SD', perfdata['rawx'])
self.assertIn('upload.RSD', perfdata['rawx'])
self.assertIn('overall', perfdata['rawx'])
perfdata.clear()
self.api.object_delete(self.account, container, obj, perfdata=perfdata)
self.assertIn('proxy', perfdata)
self.assertIn('resolve', perfdata['proxy'])
self.assertIn('meta2', perfdata['proxy'])
self.assertIn('overall', perfdata['proxy'])
def test_container_get_properties(self):
name = random_str(32)
# container_get_properties on unknown container
self.assertRaises(
exc.NoSuchContainer, self.api.container_get_properties,
self.account, name)
res = self._create(name)
self.assertEqual(res, True)
# container_get_properties on existing container
data = self.api.container_get_properties(self.account, name)
self.assertEqual(data['properties'], {})
self.assertIsNot(data['system'], None)
self.assertIn("sys.user.name", data['system'])
# container_get_properties
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
self._set_properties(name, metadata)
data = self.api.container_get_properties(self.account, name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
# container_get_properties on deleted container
self.assertRaises(
exc.NoSuchContainer, self.api.container_get_properties,
self.account, name)
def _test_admin_debug_on_srvtype(self, srvtype):
params = {
'ref': random_str(64),
'acct': random_str(64),
'type': srvtype
}
self.request('POST', self._url('admin/debug'), params=params)
def test_container_get_properties(self):
name = random_str(32)
# container_get_properties on unknown container
self.assertRaises(
exc.NoSuchContainer, self.api.container_get_properties,
self.account, name)
res = self._create(name)
self.assertEqual(res, True)
# container_get_properties on existing container
data = self.api.container_get_properties(self.account, name)
self.assertEqual(data['properties'], {})
# container_get_properties
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
self._set_properties(name, metadata)
data = self.api.container_get_properties(self.account, name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
# container_get_properties on deleted container
self.assertRaises(
exc.NoSuchContainer, self.api.container_get_properties,
self.account, name)
def setUp(self):
super(TestBlobConverter, self).setUp()
self.container = random_str(16)
self.path = random_str(16)
self.api = ObjectStorageApi(self.ns)
self.api.container_create(self.account, self.container)
_, chunks = self.api.container.content_prepare(
self.account, self.container, self.path, 1)
services = self.conscience.all_services('rawx')
self.rawx_volumes = dict()
for rawx in services:
tags = rawx['tags']
service_id = tags.get('tag.service_id', None)
if service_id is None:
service_id = rawx['addr']
volume = tags.get('tag.vol', None)
self.rawx_volumes[service_id] = volume
self.api.object_create(
self.account, self.container, obj_name=self.path, data="chunk")
meta, self.chunks = self.api.object_locate(
self.account, self.container, self.path)
self.version = meta['version']
self.content_id = meta['id']
def test_account_refresh(self):
# account_refresh on unknown account
account = random_str(32)
self.assertRaises(exc.NoSuchAccount, self.api.account_refresh, account)
# account_refresh on existing account
self.api.account_create(account)
self.api.account_refresh(account)
time.sleep(0.5) # ensure container event have been processed
res = self.api.account_show(account)
self.assertEqual(res["bytes"], 0)
self.assertEqual(res["objects"], 0)
self.assertEqual(res["containers"], 0)
name = random_str(32)
self.api.object_create(account, name, data="data", obj_name=name)
time.sleep(0.5) # ensure container event have been processed
self.api.account_refresh(account)
time.sleep(0.5) # ensure container event have been processed
res = self.api.account_show(account)
self.assertEqual(res["bytes"], 4)
self.assertEqual(res["objects"], 1)
self.assertEqual(res["containers"], 1)
self.api.object_delete(account, name, name)
time.sleep(0.5) # ensure container event have been processed
self.api.container_delete(account, name)
time.sleep(0.5) # ensure container event have been processed
self.api.account_delete(account)
# account_refresh on deleted account
self.assertRaises(exc.NoSuchAccount, self.api.account_refresh, account)
def test_create_metadata(self):
api = self.api
resp = FakeAPIResponse()
resp.status_code = 201
api._direct_request = Mock(return_value=(resp, None))
metadata = {}
k1 = random_str(32)
v1 = random_str(32)
k2 = random_str(32)
v2 = random_str(32)
metadata[k1] = v1
metadata[k2] = v2
api.create(self.account, self.name, properties=metadata)
uri = "%s/reference/create" % self.uri_base
params = {'acct': self.account, 'ref': self.name}
data = json.dumps({'properties': metadata})
api._direct_request.assert_called_once_with('POST',
uri,
params=params,
data=data)
def test_lock_unlock(self):
# lock without who
resp = self.app.post("/v1/rdir/admin/lock",
query_string={'vol': self.volume},
data=json.dumps({}))
self.assertEqual(resp.status_code, 400)
# lock
who = random_str(64)
data = {'who': who}
resp = self.app.post("/v1/rdir/admin/lock",
query_string={'vol': self.volume},
data=json.dumps(data))
self.assertEqual(resp.status_code, 204)
# double lock
data = {'who': random_str(64)}
resp = self.app.post("/v1/rdir/admin/lock",
query_string={'vol': self.volume},
data=json.dumps(data))
self.assertEqual(resp.status_code, 403)
self.assertEqual(resp.data, "Already locked by %s" % who)
# unlock
resp = self.app.post("/v1/rdir/admin/unlock",
query_string={'vol': self.volume})
self.assertEqual(resp.status_code, 204)
def test_container_get_properties_filtered(self):
self.skipTest("Server side properties filtering not implemented")
name = random_str(32)
res = self._create(name)
self.assertEqual(res, True)
# container_get_properties on existing container
data = self.api.container_get_properties(self.account, name)
self.assertEqual(data['properties'], {})
# container_get_properties
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
self._set_properties(name, metadata)
# container_get_properties specify key
key = metadata.keys().pop(0)
data = self.api.container_get_properties(self.account, name, [key])
self.assertEqual({key: metadata[key]}, data['properties'])
# clean
self._clean(name, True)
def test_container_list(self):
resp = FakeApiResponse()
name = random_str(32)
marker = random_str(32)
delimiter = random_str(32)
end_marker = random_str(32)
prefix = random_str(32)
limit = random.randint(1, 1000)
body = {"listing": [[name, 0, 0, 0]]}
fake_endpoint = 'fake_endpoint'
self.api.account._direct_request = Mock(return_value=(resp, body))
self.api.account._get_account_addr = Mock(return_value=fake_endpoint)
containers = self.api.container_list(self.account,
limit=limit,
marker=marker,
prefix=prefix,
delimiter=delimiter,
end_marker=end_marker,
headers=self.headers)
params = {
"id": self.account,
"prefix": prefix,
"delimiter": delimiter,
"marker": marker,
"end_marker": end_marker,
"limit": limit
}
uri = "http://%s/v1.0/account/containers" % fake_endpoint
self.api.account._direct_request.assert_called_once_with(
'GET', uri, params=params, headers=self.headers)
self.assertEqual(len(containers), 1)
def test_container_snapshot(self):
# Snapshot should reply the name of the snapshot on success
opts = self.get_opts([], 'json')
output = self.openio('container snapshot ' + self.NAME + opts)
output = self.json_loads(output)[0]
self.assertEqual(output['Status'], "OK")
# Snapshot should reply Missing container on non existant container
self.assertRaises(CommandFailed,
self.openio,
('container snapshot Should_not_exist' + opts))
# Use specified name
dst_account = random_str(16)
dst_container = random_str(16)
opts += " --dst-account " + dst_account
opts += " --dst-container " + dst_container
output = self.openio('container snapshot ' + self.NAME + opts)
output = self.json_loads(output)[0]
self.assertEqual(output['Account'], dst_account)
self.assertEqual(output['Container'], dst_container)
self.assertEqual(output['Status'], "OK")
# Snapshot should reply Container already exists when using already
# specified name
self.assertRaises(CommandFailed,
self.openio,
('container snapshot ' + self.NAME + opts))
def _test_object_link(self, with_cid=False):
cont_name = random_str(8)
obj_name = random_str(8)
lk_name = obj_name + '-link'
cid_opt = ''
output = self.openio('container create ' + cont_name)
if with_cid:
cont_name = self._get_cid_from_name(cont_name)
cid_opt = '--cid '
with tempfile.NamedTemporaryFile() as myfile:
myfile.write(b'something')
myfile.flush()
output = self.openio('object create ' + cid_opt + cont_name + ' ' +
myfile.name + ' --name ' + obj_name +
' -f json')
output = self.openio('object show -f json ' + cid_opt + cont_name +
' ' + obj_name)
output = self.json_loads(output)
self.assertEqual(output['object'], obj_name)
output = self.openio('object link ' + cid_opt + cont_name + ' ' +
obj_name + ' ' + lk_name)
self.assertEqual(output, '')
output = self.openio('object show -f json ' + cid_opt + cont_name +
' ' + lk_name)
output = self.json_loads(output)
self.assertEqual(output['object'], lk_name)
def test_object_show(self):
api = self.api
name = random_str(32)
size = random.randint(1, 1000)
content_hash = random_str(32)
content_type = random_str(32)
resp = FakeApiResponse()
resp.headers = {
object_headers["name"]: name,
object_headers["size"]: str(size),
object_headers["hash"]: content_hash,
object_headers["mime_type"]: content_type
}
api.container._direct_request = Mock(return_value=(resp, {
'properties': {}
}))
obj = api.object_show(self.account,
self.container,
name,
headers=self.headers)
uri = "%s/content/get_properties" % self.uri_base
params = {'acct': self.account, 'ref': self.container, 'path': name}
api.container._direct_request.assert_called_once_with(
'POST', uri, params=params, data=None, headers=self.headers)
self.assertIsNotNone(obj)
def _test_container_snapshot(self, with_cid=False):
self.wait_for_score(('meta2', ))
# Snapshot should reply the name of the snapshot on success
opts = self.get_format_opts('json')
cid_opt = ''
cname = self.NAME
if with_cid:
cid_opt = '--cid '
cname = self.CID
output = self.openio('container snapshot ' + cid_opt + cname + opts)
output = self.json_loads(output)[0]
self.assertEqual(output['Status'], "OK")
# Snapshot should reply Missing container on non existant container
self.assertRaises(CommandFailed, self.openio,
('container snapshot Should_not_exist' + opts))
# Use specified name
dst_account = 'acct-' + random_str(6)
dst_container = cname + '.snapshot-' + random_str(6)
opts += " --dst-account " + dst_account
opts += " --dst-container " + dst_container
output = self.openio('container snapshot ' + cid_opt + cname + opts)
output = self.json_loads(output)[0]
self.assertEqual(output['Account'], dst_account)
self.assertEqual(output['Container'], dst_container)
self.assertEqual(output['Status'], "OK")
# Snapshot should reply Container already exists when using already
# specified name
self.assertRaises(CommandFailed, self.openio,
('container snapshot ' + cid_opt + cname + opts))
def test_download_with_stopped_rawx(self):
container = random_str(8)
obj = random_str(8)
expected_data = random_data(10)
chunks, _, _, meta = self.storage.object_create_ext(
self.account, container, obj_name=obj, data=expected_data,
policy='EC')
storage_method = STORAGE_METHODS.load(meta['chunk_method'])
sorted_chunks = _sort_chunks(chunks, storage_method.ec)
sorted_present_chunks = sorted_chunks.copy()
try:
for i in range(storage_method.ec_nb_parity):
data = ''
for pos in range(len(sorted_chunks)):
if pos == 0:
chunk = random.choice(sorted_present_chunks[pos])
sorted_present_chunks[pos].remove(chunk)
gridinit_key = self.service_to_gridinit_key(
urlparse(chunk['url']).netloc, 'rawx')
self._service(gridinit_key, 'stop')
resp = self._download_metachunk(meta, sorted_chunks[pos])
self.assertEqual(200, resp.status)
data += resp.data
self.assertEqual(expected_data, data)
chunk = random.choice(sorted_present_chunks[0])
sorted_present_chunks[0].remove(chunk)
gridinit_key = self.service_to_gridinit_key(
urlparse(chunk['url']).netloc, 'rawx')
self._service(gridinit_key, 'stop')
resp = self._download_metachunk(meta, sorted_chunks[pos])
self.assertEqual(500, resp.status)
finally:
self._service('@rawx', 'start')
def setUp(self):
super(TestBlobRebuilder, self).setUp()
self.container = random_str(16)
self.cid = cid_from_name(self.account, self.container)
self.path = random_str(16)
self.api = ObjectStorageApi(self.ns)
self.blob_client = BlobClient(self.conf)
self.api.container_create(self.account, self.container)
_, chunks = self.api.container.content_prepare(
self.account, self.container, self.path, size=1)
if len(chunks) < 2:
self.skipTest("need at least 2 chunks to run")
services = self.conscience.all_services('rawx')
self.rawx_volumes = dict()
for rawx in services:
tags = rawx['tags']
service_id = tags.get('tag.service_id', None)
if service_id is None:
service_id = rawx['addr']
volume = tags.get('tag.vol', None)
self.rawx_volumes[service_id] = volume
self.api.object_create(
self.account, self.container, obj_name=self.path, data="chunk")
meta, self.chunks = self.api.object_locate(
self.account, self.container, self.path)
self.version = meta['version']
self.content_id = meta['id']
def test_bucket_show_with_account_refresh(self):
account = 'myaccount-' + random_str(4).lower()
opts = self.get_format_opts(fields=('Name', ))
cname = 'mybucket-' + random_str(4).lower()
output = self.openio(
'--oio-account %s container create --bucket-name %s %s %s' %
(account, cname, cname, opts))
self.assertOutput(cname + '\n', output)
self.wait_for_event('oio-preserved',
fields={'user': cname},
types=(EventTypes.CONTAINER_STATE, ))
with tempfile.NamedTemporaryFile() as file_:
file_.write(b'test')
file_.flush()
output = self.openio(
'--oio-account %s object create %s %s --name test' %
(account, cname, file_.name))
self.wait_for_event('oio-preserved',
fields={'user': cname},
types=(EventTypes.CONTAINER_STATE, ))
opts = self.get_format_opts(fields=('account', 'bytes', 'objects'))
output = self.openio('bucket show ' + cname + opts)
self.assertEqual(account + '\n4\n1\n', output)
output = self.openio('account refresh ' + account)
self.wait_for_event('oio-preserved',
fields={'user': cname},
types=(EventTypes.CONTAINER_STATE, ))
output = self.openio('bucket show ' + cname + opts)
self.assertEqual(account + '\n4\n1\n', output)
def test_container_get_properties_filtered(self):
self.skipTest("Server side properties filtering not implemented")
name = random_str(32)
res = self._create(name)
self.assertEqual(res, True)
# container_get_properties on existing container
data = self.api.container_get_properties(self.account, name)
self.assertEqual(data['properties'], {})
# container_get_properties
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
self._set_properties(name, metadata)
# container_get_properties specify key
key = metadata.keys().pop(0)
data = self.api.container_get_properties(self.account, name, [key])
self.assertEqual({key: metadata[key]}, data['properties'])
# clean
self._clean(name, True)
def test_object_fetch_perfdata(self):
perfdata = dict()
container = random_str(8)
obj = random_str(8)
self.api.object_create(self.account, container, obj_name=obj, data=obj)
meta, chunks = self.api.object_locate(self.account, container, obj)
stg_method = STORAGE_METHODS.load(meta['chunk_method'])
_, stream = self.api.object_fetch(self.account, container, obj,
perfdata=perfdata)
self.assertIn('proxy', perfdata)
self.assertIn('resolve', perfdata['proxy'])
self.assertIn('meta2', perfdata['proxy'])
self.assertIn('overall', perfdata['proxy'])
self.assertNotIn('ttfb', perfdata)
self.assertNotIn('ttlb', perfdata)
buf = b''.join(stream)
self.assertEqual(obj, buf)
self.assertIn('rawx', perfdata)
if stg_method.ec:
self.assertIn('ec', perfdata['rawx'])
nb_chunks_to_read = 0
for chunk in chunks:
if chunk['url'] in perfdata['rawx']:
nb_chunks_to_read += 1
self.assertLessEqual(stg_method.min_chunks_to_read,
nb_chunks_to_read)
self.assertIn('overall', perfdata['rawx'])
self.assertIn('ttfb', perfdata)
self.assertIn('ttlb', perfdata)
self.api.object_delete(self.account, container, obj)
def test_refresh_bucket(self):
account_id = random_str(16)
account_key = 'account:%s' % account_id
bucket = random_str(16)
self.assertEqual(self.backend.create_account(account_id), account_id)
total_bytes = 0
total_objects = 0
total_damaged_objects = 0
total_missing_chunks = 0
# 10 containers with bytes and objects
for i in range(10):
name = "container%d" % i
mtime = Timestamp().normal
nb_bytes = random.randrange(100)
total_bytes += nb_bytes
nb_objets = random.randrange(100)
total_objects += nb_objets
damaged_objects = random.randrange(100)
total_damaged_objects += damaged_objects
missing_chunks = random.randrange(100)
total_missing_chunks += missing_chunks
self.backend.update_container(account_id,
name,
mtime,
0,
nb_objets,
nb_bytes,
damaged_objects,
missing_chunks,
bucket_name=bucket)
bkey = self.backend.bkey(bucket)
# change values
self.backend.conn.hset(bkey, 'bytes', 1)
self.backend.conn.hset(bkey, 'objects', 2)
self.backend.conn.hset(bkey, 'damaged_objects', 3)
self.backend.conn.hset(bkey, 'missing_chunks', 4)
self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'1')
self.assertEqual(self.backend.conn.hget(bkey, 'objects'), b'2')
self.assertEqual(self.backend.conn.hget(bkey, 'damaged_objects'), b'3')
self.assertEqual(self.backend.conn.hget(bkey, 'missing_chunks'), b'4')
# force pagination
self.backend.refresh_bucket(bucket, batch_size=6)
conn = self.backend.conn
self.assertEqual(
conn.hget(account_key, 'bytes').decode('utf-8'), str(total_bytes))
self.assertEqual(
conn.hget(account_key, 'objects').decode('utf-8'),
str(total_objects))
self.assertEqual(
conn.hget(account_key, 'damaged_objects').decode('utf-8'),
str(total_damaged_objects))
self.assertEqual(
conn.hget(account_key, 'missing_chunks').decode('utf-8'),
str(total_missing_chunks))
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.test_dir = self.conf['sds_path']
rawx_num, rawx_path, rawx_addr = self.get_service_url('rawx')
self.rawx = 'http://' + rawx_addr
self.h = hashlib.new('md5')
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = random_str(8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = random_id(64)
self.data = random_str(1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(random_str(6), len(self.data),
self.url_rand, 1)
self.content.id_container = cid_from_name(self.account,
self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0,
self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {
"hash": self.chunk.md5,
"pos": "0",
"size": self.chunk.size,
"url": self.chunk_url
}
chunk_meta = {
'content_path': self.content.path,
'container_id': self.content.id_container,
'chunk_method': 'plain/nb_copy=3',
'policy': 'TESTPOLICY',
'id': '0000',
'version': 1,
'chunk_id': self.chunk.id_chunk,
'chunk_pos': self.chunk.pos,
'chunk_hash': self.chunk.md5,
}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = self.test_dir + '/data/' + self.namespace + \
'-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk
self.bad_container_id = '0' * 64
def test_container_not_found(self):
account = random_str(10)
container = random_str(10)
opts = self.get_format_opts()
self.assertRaises(
CommandFailed, self.openio_admin,
'container locate --oio-account %s %s %s'
% (account, container, opts))
def test_object_create_long_name(self):
"""Create an objet whose name has the maximum length allowed"""
cname = random_str(16)
path = random_str(1023)
self.api.object_create(self.account,
cname,
data="1" * 128,
obj_name=path)
def setUp(self):
super(ItemRebuildTest, self).setUp()
self.rawx_services = None
self.container = "item_rebuild_container" + random_str(4)
self.obj_name = "item_rebuild_obj_" + random_str(4)
self.beanstalkd0.drain_tube('oio-preserved')
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf["namespace"]
self.account = self.conf["account"]
self.test_dir = self.conf["sds_path"]
rawx_num, rawx_path, rawx_addr = self.get_service_url("rawx")
self.rawx = "http://" + rawx_addr
self.h = hashlib.new("md5")
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = random_str(8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = random_id(64)
self.data = random_str(1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(random_str(6), len(self.data), self.url_rand, 1)
self.content.id_container = cid_from_name(self.account, self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0, self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0", "size": self.chunk.size, "url": self.chunk_url}
chunk_meta = {
"content_path": self.content.path,
"container_id": self.content.id_container,
"chunk_method": "plain/nb_copy=3",
"policy": "TESTPOLICY",
"id": "0000",
"version": 1,
"chunk_id": self.chunk.id_chunk,
"chunk_pos": self.chunk.pos,
"chunk_hash": self.chunk.md5,
}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = (
self.test_dir
+ "/data/"
+ self.namespace
+ "-rawx-1/"
+ self.chunk.id_chunk[0:3]
+ "/"
+ self.chunk.id_chunk
)
self.bad_container_id = "0" * 64
def _sized_prop(self, ksize=None, vsize=None):
if ksize:
pkey = random_str(ksize)
else:
pkey = "key"
if vsize:
pval = random_str(vsize)
else:
pval = "value"
return pkey, pval
def _meta2_record(self):
return {
"container_id":
random_id(64),
"container_url":
"{0}/{1}/{2}".format(random_str(12), random_str(12),
random_str(12)),
"mtime":
int(time.time())
}
def test_set_properties(self):
api = self.api
properties = {random_str(64): random_str(64)}
resp = FakeAPIResponse()
api._request = Mock(return_value=(resp, None))
api.set_properties(self.account, self.name, properties)
uri = "%s/reference/set_properties" % self.uri_base
params = {'acct': self.account, 'ref': self.name}
data = json.dumps({'properties': properties})
api._request.assert_called_once_with(
'POST', uri, data=data, params=params, headers=None)
def _test_object_set_properties(self, with_cid=False):
cont_name = random_str(8)
obj_name = random_str(8)
cid_opt = ''
output = self.openio('container create ' + cont_name)
if with_cid:
cont_name = self._get_cid_from_name(cont_name)
cid_opt = '--cid '
with tempfile.NamedTemporaryFile() as myfile:
myfile.write(b'something')
myfile.flush()
output = self.openio('object create ' + cid_opt + cont_name + ' ' +
myfile.name + ' --name ' + obj_name +
' -f json')
output = self.openio('object show -f json ' + cid_opt + cont_name +
' ' + obj_name)
output = self.json_loads(output)
self.assertEqual(obj_name, output['object'])
output = self.openio('object set ' + cid_opt + cont_name + ' ' +
obj_name +
' --property test1=1 --property test2=2')
self.assertEqual(output, '')
output = self.openio('object show -f json ' + cid_opt + cont_name +
' ' + obj_name)
output = self.json_loads(output)
self.assertEqual(obj_name, output['object'])
self.assertEqual('1', output['meta.test1'])
self.assertEqual('2', output['meta.test2'])
output = self.openio('object set ' + cid_opt + cont_name + ' ' +
obj_name + ' --property test3=3')
self.assertEqual(output, '')
output = self.openio('object show -f json ' + cid_opt + cont_name +
' ' + obj_name)
output = self.json_loads(output)
self.assertEqual(obj_name, output['object'])
self.assertEqual('1', output['meta.test1'])
self.assertEqual('2', output['meta.test2'])
self.assertEqual('3', output['meta.test3'])
output = self.openio('object set ' + cid_opt + cont_name + ' ' +
obj_name + ' --clear' + ' --property test4=4')
self.assertEqual(output, '')
output = self.openio('object show -f json ' + cid_opt + cont_name +
' ' + obj_name)
output = self.json_loads(output)
self.assertEqual(obj_name, output['object'])
self.assertNotIn('meta.test1', output)
self.assertNotIn('meta.test2', output)
self.assertNotIn('meta.test3', output)
self.assertEqual('4', output['meta.test4'])
def test_set_properties(self):
api = self.api
properties = {random_str(64): random_str(64)}
resp = FakeApiResponse()
api._direct_request = Mock(return_value=(resp, None))
api.set_properties(self.account, self.name, properties)
uri = "%s/reference/set_properties" % self.uri_base
params = {'acct': self.account, 'ref': self.name}
data = json.dumps({'properties': properties})
api._direct_request.assert_called_once_with(
'POST', uri, data=data, params=params)
def test_container_locate(self):
container = random_str(10)
account = random_str(10)
opts = self.get_format_opts()
output = self.openio('container create --oio-account %s %s %s'
% (account, container, opts))
self.assertOutput('%s True' % container, output.strip())
output = self.openio_admin('container locate --oio-account %s %s %s' %
(account, container, opts))
cid = cid_from_name(account, container)
self._test_container_check_output(output, cid, account, container)
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.test_dir = self.conf['sds_path']
rawx_num, rawx_path, rawx_addr = self.get_service_url('rawx')
self.rawx = 'http://' + rawx_addr
self.h = hashlib.new('md5')
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = random_str(8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = random_id(64)
self.data = random_str(1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(
random_str(6), len(self.data), self.url_rand, 1)
self.content.id_container = cid_from_name(
self.account, self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0,
self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0",
"size": self.chunk.size,
"url": self.chunk_url}
chunk_meta = {'content_path': self.content.path,
'container_id': self.content.id_container,
'content_chunkmethod': 'plain/nb_copy=3',
'content_policy': 'TESTPOLICY',
'content_id': '0000',
'content_version': 1,
'chunk_id': self.chunk.id_chunk,
'chunk_pos': self.chunk.pos}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = self.test_dir + '/data/' + self.namespace + \
'-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk
self.bad_container_id = '0'*64
def test_object_delete_not_found(self):
api = self.api
name = random_str(32)
api._request = Mock(side_effect=exceptions.NotFound("No object"))
self.assertRaises(
exceptions.NoSuchObject, api.object_delete, self.account,
self.container, name)
def test_object_create_no_content_length(self):
api = self.api
name = random_str(32)
f = Mock()
self.assertRaises(
exceptions.MissingContentLength, api.object_create, self.account,
self.container, f, obj_name=name)
def setUp(self):
self.api = FakeStorageAPI("NS", "http://1.2.3.4:8000")
self.account = "test"
self.container = "fake"
self.headers = {"x-req-id": random_str(32)}
self.policy = "THREECOPIES"
self.uri_base = "NS"
def test_chunk_corrupted(self):
self.init_content()
with open(self.chunk_path, "w") as f:
f.write(random_str(1280))
self.assertRaises(exc.CorruptedChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_create_properties(self):
name = random_str(32)
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
res = self._create(name, metadata)
self.assertEqual(res, True)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
def test_chunk_bad_size(self):
self.init_content()
with open(self.chunk_path, "w") as f:
f.write(random_str(320))
self.assertRaises(exc.FaultyChunk, self.auditor.chunk_audit,
self.chunk_path)
def test_container_update(self):
api = self.api
name = random_str(32)
key = random_str(32)
value = random_str(32)
meta = {key: value}
resp = FakeAPIResponse()
api._request = Mock(return_value=(resp, None))
api.container_update(self.account, name, meta, headers=self.headers)
data = json.dumps(meta)
uri = "%s/container/set_properties" % self.uri_base
params = {'acct': self.account, 'ref': name}
api._request.assert_called_once_with(
'POST', uri, data=data, params=params, headers=self.headers)
def setUp(self):
self.endpoint = "http://1.2.3.4:8000"
self.api = FakeDirectoryAPI("NS", self.endpoint)
self.account = "AUTH_test"
self.headers = {"x-req-id": random_id(64)}
self.uri_base = "NS"
self.name = random_str(64)
def test_handle_container_not_found(self):
@handle_container_not_found
def test(self, account, container):
raise exceptions.NotFound("No container")
container = random_str(32)
self.assertRaises(
exceptions.NoSuchContainer, test, self, self.account, container)
def test_container_create_exist(self):
api = self.api
resp = FakeAPIResponse()
resp.status_code = 201
api._request = Mock(return_value=(resp, None))
name = random_str(32)
result = api.container_create(self.account, name)
self.assertEqual(result, False)
def test_handle_object_not_found(self):
@handle_object_not_found
def test(self, account, container, obj):
raise exceptions.NotFound("No object")
obj = random_str(32)
self.assertRaises(
exceptions.NoSuchObject, test, self, self.account, self.container,
obj)
def test_container_delete_not_empty(self):
api = self.api
api._request = Mock(side_effect=exceptions.Conflict(""))
api.directory.unlink = Mock(return_value=None)
name = random_str(32)
self.assertRaises(
exceptions.ContainerNotEmpty, api.container_delete, self.account,
name)
def test_create(self):
name = random_str(32)
res = self._create(name)
self.assertEqual(res, True)
# second create
res = self._create(name)
self.assertEqual(res, False)
# clean
self._delete(name)
def test_renew(self):
api = self.api
service_type = random_str(32)
resp = FakeAPIResponse()
api._request = Mock(return_value=(resp, None))
api.renew(self.account, self.name, service_type)
uri = "%s/reference/renew" % self.uri_base
params = {'acct': self.account, 'ref': self.name,
'type': service_type}
api._request.assert_called_once_with(
'POST', uri, params=params, headers=None)
def test_object_show(self):
api = self.api
name = random_str(32)
size = random.randint(1, 1000)
content_hash = random_str(32)
content_type = random_str(32)
resp = FakeAPIResponse()
resp.headers = {object_headers["name"]: name,
object_headers["size"]: size,
object_headers["hash"]: content_hash,
object_headers["mime_type"]: content_type}
api._request = Mock(return_value=(resp, {'properties': {}}))
obj = api.object_show(
self.account, self.container, name, headers=self.headers)
uri = "%s/content/get_properties" % self.uri_base
params = {'acct': self.account, 'ref': self.container,
'path': name}
api._request.assert_called_once_with(
'POST', uri, params=params, headers=self.headers)
self.assertIsNotNone(obj)
def test_container_list(self):
resp = FakeAPIResponse()
name = random_str(32)
marker = random_str(32)
delimiter = random_str(32)
end_marker = random_str(32)
prefix = random_str(32)
limit = random.randint(1, 1000)
body = {"listing": [[name, 0, 0, 0]]}
self.api._request = Mock(return_value=(resp, body))
self.api._get_service_url = Mock(return_value='fake_endpoint')
containers, meta = self.api.container_list(
self.account, limit=limit, marker=marker, prefix=prefix,
delimiter=delimiter, end_marker=end_marker, headers=self.headers)
params = {"id": self.account, "prefix": prefix, "delimiter": delimiter,
"marker": marker, "end_marker": end_marker, "limit": limit}
uri = "v1.0/account/containers"
self.api._request.assert_called_once_with(
'GET', uri, endpoint='fake_endpoint', params=params,
headers=self.headers)
self.assertEqual(len(containers), 1)
def test_force(self):
api = self.api
service_type = random_str(32)
services = {'seq': 1, 'type': service_type, 'host': '127.0.0.1:8000'}
resp = FakeAPIResponse()
api._request = Mock(return_value=(resp, None))
api.force(self.account, self.name, service_type, services)
uri = "%s/reference/force" % self.uri_base
params = {'acct': self.account, 'ref': self.name,
'type': service_type}
data = json.dumps(services)
api._request.assert_called_once_with(
'POST', uri, data=data, params=params, headers=None)
def test_set_properties(self):
name = random_str(32)
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
# set_properties on unknown reference
self.assertRaises(
exc.NotFound, self.api.set_properties, self.account, name,
metadata)
res = self._create(name)
self.assertEqual(res, True)
# set_properties on existing reference
self.api.set_properties(self.account, name, metadata)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# set_properties
key = random_str(32)
value = random_str(32)
metadata2 = {key: value}
self._set_properties(name, metadata2)
metadata.update(metadata2)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# set_properties overwrite key
key = metadata.keys().pop(0)
value = random_str(32)
metadata3 = {key: value}
metadata.update(metadata3)
self.api.set_properties(self.account, name, metadata3)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
# set_properties on deleted reference
self.assertRaises(
exc.NotFound, self.api.set_properties, self.account, name,
metadata)
def test_get(self):
# get on unknown reference
name = random_str(32)
self.assertRaises(exc.NotFound, self.api.get, self.account, name)
self._create(name)
# get on existing reference
res = self.api.get(self.account, name)
self.assertIsNot(res['dir'], None)
self.assertIsNot(res['srv'], None)
self._delete(name)
# get on deleted reference
self.assertRaises(exc.NotFound, self.api.get, self.account, name)
def test_container_delete(self):
api = self.api
resp = FakeAPIResponse()
resp.status_code = 204
api._request = Mock(return_value=(resp, None))
api.directory.unlink = Mock(return_value=None)
name = random_str(32)
api.container_delete(self.account, name, headers=self.headers)
uri = "%s/container/destroy" % self.uri_base
params = {'acct': self.account, 'ref': name}
api._request.assert_called_once_with(
'POST', uri, params=params, headers=self.headers)
def test_create_metadata(self):
api = self.api
resp = FakeAPIResponse()
resp.status_code = 201
api._request = Mock(return_value=(resp, None))
metadata = {}
k1 = random_str(32)
v1 = random_str(32)
k2 = random_str(32)
v2 = random_str(32)
metadata[k1] = v1
metadata[k2] = v2
api.create(self.account, self.name, metadata)
uri = "%s/reference/create" % self.uri_base
params = {'acct': self.account, 'ref': self.name}
data = json.dumps({'properties': metadata})
api._request.assert_called_once_with(
'POST', uri, params=params, data=data, headers=None)
def test_container_show(self):
api = self.api
resp = FakeAPIResponse()
name = random_str(32)
cont_size = random.randint(1, 1000)
resp.headers = {
container_headers["size"]: cont_size
}
api._request = Mock(return_value=(resp, {}))
info = api.container_show(self.account, name, headers=self.headers)
uri = "%s/container/get_properties" % self.uri_base
params = {'acct': self.account, 'ref': name}
api._request.assert_called_once_with(
'POST', uri, params=params, headers=self.headers)
self.assertEqual(info, {})
def test_get_properties(self):
name = random_str(32)
# get_properties on unknown reference
self.assertRaises(
exc.NotFound, self.api.get_properties, self.account, name)
res = self._create(name)
self.assertEqual(res, True)
# get_properties on existing reference
data = self.api.get_properties(self.account, name)
self.assertEqual(data['properties'], {})
# get_properties
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
self._set_properties(name, metadata)
data = self.api.get_properties(self.account, name)
self.assertEqual(data['properties'], metadata)
# get_properties specify key
key = metadata.keys().pop(0)
data = self.api.get_properties(self.account, name, [key])
self.assertEqual(data['properties'], {key: metadata[key]})
# clean
self._clean(name, True)
# get_properties on deleted reference
self.assertRaises(
exc.NotFound, self.api.get_properties, self.account, name)
def test_container_create(self):
api = self.api
resp = FakeAPIResponse()
resp.status_code = 204
api._request = Mock(return_value=(resp, None))
name = random_str(32)
result = api.container_create(self.account, name, headers=self.headers)
self.assertEqual(result, True)
uri = "%s/container/create" % self.uri_base
params = {'acct': self.account, 'ref': name}
self.headers['x-oio-action-mode'] = 'autocreate'
api._request.assert_called_once_with(
'POST', uri, params=params, headers=self.headers)