def worker_container():
proxy = ObjectStorageApi(NS)
while True:
try:
name = QUEUE.get(timeout=TIMEOUT)
except eventlet.queue.Empty:
break
while True:
if VERBOSE:
print("Deleting", name)
try:
proxy.container_delete(ACCOUNT, name)
COUNTERS.add(1, 0)
break
except Exception as ex:
if "Election failed" in str(ex):
# wait default Election wait delay
ELECTIONS.add(1, 0)
time.sleep(20)
continue
print("Container %s: %s" % (name, str(ex)), file=sys.stderr)
break
QUEUE.task_done()
class TestObjectStorageApiPerformance(BaseTestCase):
def setUp(self):
super(TestObjectStorageApiPerformance, self).setUp()
self.api = ObjectStorageApi(self.ns,
endpoint=self.uri,
source_address=('127.0.0.8', 0))
self.created = list()
self.containers = set()
def tearDown(self):
super(TestObjectStorageApiPerformance, self).tearDown()
for ct, name in self.created:
try:
self.api.object_delete(self.account, ct, name)
self.containers.add(ct)
except Exception:
logging.exception("Failed to delete %s/%s/%s//%s", self.ns,
self.account, ct, name)
for ct in self.containers:
try:
self.api.container_delete(self.account, ct)
except Exception:
logging.exception('Failed to delete %s/%s/%s', self.ns,
self.account, ct)
def test_object_create_32_md5_checksum(self):
container = self.__class__.__name__ + random_str(8)
for i in range(32):
obj = "obj-%03d" % i
self.api.object_create(self.account,
container,
obj_name=obj,
data=obj,
chunk_checksum_algo='md5')
self.created.append((container, obj))
def test_object_create_32_no_checksum(self):
container = self.__class__.__name__ + random_str(8)
for i in range(32):
obj = "obj-%03d" % i
self.api.object_create(self.account,
container,
obj_name=obj,
data=obj,
chunk_checksum_algo=None)
self.created.append((container, obj))
def test_object_list_empty_container(self):
"""
Ensure object listing of an empty container takes less than 35ms.
"""
container = self.__class__.__name__ + random_str(8)
self.api.container_create(self.account, container)
self.containers.add(container)
for _ in range(8):
start = monotonic_time()
self.api.object_list(self.account, container)
duration = monotonic_time() - start
logging.info("Object list took %.6fs", duration)
self.assertLess(duration, 0.035)
class ObjectStorageApiTestBase(BaseTestCase):
def setUp(self):
super(ObjectStorageApiTestBase, self).setUp()
self.api = ObjectStorageApi(self.ns, endpoint=self.uri)
self.created = list()
def tearDown(self):
super(ObjectStorageApiTestBase, self).tearDown()
for ct, name in self.created:
try:
self.api.object_delete(self.account, ct, name)
except Exception:
logging.exception("Failed to delete %s/%s/%s//%s", self.ns,
self.account, ct, name)
def _create(self, name, metadata=None):
return self.api.container_create(self.account,
name,
properties=metadata)
def _delete(self, name):
self.api.container_delete(self.account, name)
def _clean(self, name, clear=False):
if clear:
# must clean properties before
self.api.container_del_properties(self.account, name, [])
self._delete(name)
def _get_properties(self, name, properties=None):
return self.api.container_get_properties(self.account,
name,
properties=properties)
def _set_properties(self, name, properties=None):
return self.api.container_set_properties(self.account,
name,
properties=properties)
def _upload_empty(self, container, *objs, **kwargs):
"""Upload empty objects to `container`"""
for obj in objs:
self.api.object_create(self.account,
container,
obj_name=obj,
data="",
**kwargs)
self.created.append((container, obj))
class TestObjectStorageApiPerformance(BaseTestCase):
def setUp(self):
super(TestObjectStorageApiPerformance, self).setUp()
self.api = ObjectStorageApi(self.ns, endpoint=self.uri)
self.created = list()
def tearDown(self):
super(TestObjectStorageApiPerformance, self).tearDown()
containers = set()
for ct, name in self.created:
try:
self.api.object_delete(self.account, ct, name)
containers.add(ct)
except Exception:
logging.exception("Failed to delete %s/%s/%s//%s",
self.ns, self.account, ct, name)
for ct in containers:
try:
self.api.container_delete(self.account, ct)
except Exception:
logging.exception('Failed to delete %s/%s/%s',
self.ns, self.account, ct)
def test_object_create_32_md5_checksum(self):
container = self.__class__.__name__ + random_str(8)
for i in range(32):
obj = "obj-%03d" % i
self.api.object_create(self.account, container,
obj_name=obj, data=obj,
chunk_checksum_algo='md5')
self.created.append((container, obj))
def test_object_create_32_no_checksum(self):
container = self.__class__.__name__ + random_str(8)
for i in range(32):
obj = "obj-%03d" % i
self.api.object_create(self.account, container,
obj_name=obj, data=obj,
chunk_checksum_algo=None)
self.created.append((container, obj))
def main():
args = options()
global ACCOUNT, PROXY
ACCOUNT = args.account
PROXY = ObjectStorageApi("OPENIO")
args.path = args.path.rstrip('/')
if '/' in args.path:
bucket, path = args.path.split('/', 1)
else:
bucket = args.path
path = ""
containers = []
_bucket = container_hierarchy(bucket, path)
# we don't use placeholders, we use prefix path as prefix
for entry in full_list(prefix=container_hierarchy(bucket, path)):
name, _files, _size, _ = entry
if name != _bucket and not name.startswith(_bucket + '%2F'):
continue
if _files:
items = PROXY.object_list(ACCOUNT, name)
objs = [_item['name'] for _item in items['objects']]
PROXY.object_delete_many(ACCOUNT, name, objs=objs)
print("Deleting", len(objs), "objects")
containers.append(name)
print("We have to delete", len(containers), "containers")
for container in containers:
print("Deleting", container)
PROXY.container_delete(ACCOUNT, container)
class TestContainerDownload(BaseTestCase):
def setUp(self):
super(TestContainerDownload, self).setUp()
# FIXME: should we use direct API from BaseTestCase
# or still container.client ?
self.conn = ObjectStorageApi(self.ns)
self._streaming = 'http://' + self.get_service_url('container')[2]
self._cnt = random_container()
self._uri = self.make_uri('dump')
self._data = {}
self.conn.container_create(self.account, self._cnt)
self.raw = ""
self._slo = []
def make_uri(self, action, account=None, container=None):
account = account or self.account
container = container or self._cnt
return '%s/v1.0/container/%s?acct=%s&ref=%s' % (
self._streaming, action, account, container)
def tearDown(self):
for name in self._data:
self.conn.object_delete(self.account, self._cnt, name)
self.conn.container_delete(self.account, self._cnt)
super(TestContainerDownload, self).tearDown()
def _create_data(self,
name=gen_names,
metadata=None,
size=513,
append=False):
for idx, _name in itertools.islice(name(), 5):
mime = random.choice(MIMETYPE)
if append and size > 0:
data = gen_data(size / 2 * idx)
entry = {'data': data, 'meta': None, 'mime': mime}
self.conn.object_create(self.account,
self._cnt,
obj_name=_name,
data=data,
mime_type=mime)
data = gen_data(size / 2 * idx)
self.conn.object_create(self.account,
self._cnt,
obj_name=_name,
data=data,
mime_type=mime,
append=True)
entry['data'] += data
else:
data = gen_data(size * idx)
entry = {'data': data, 'meta': None, 'mime': mime}
self.conn.object_create(self.account,
self._cnt,
obj_name=_name,
data=data,
mime_type=mime)
if metadata:
entry['meta'] = {}
for _ in xrange(10):
key, val = metadata()
entry['meta'][key] = val
self.conn.object_update(self.account, self._cnt, _name,
entry['meta'])
self._data[_name] = entry
def _create_s3_slo(self, name=gen_names, metadata=None):
# create a fake S3 bucket with a SLO object
chunksize = 10000
parts = 5
res = []
full_data = ""
self.conn.container_create(self.account, self._cnt + '+segments')
_name = "toto"
etag = rand_str(50)
part_number = 1
for size in [chunksize] * parts + [444]:
data = gen_data(size)
res.append({
'bytes':
size,
'content_type':
'application/octect-stream',
'hash':
md5(data).hexdigest().upper(),
'last_modified':
'2017-06-21T12:42:47.000000',
'name':
'/%s+segments/%s/%s/%d' % (self._cnt, _name, etag, part_number)
})
self.conn.object_create(self.account,
"%s+segments" % self._cnt,
obj_name='%s/%s/%d' %
(_name, etag, part_number),
data=data)
full_data += data
part_number += 1
self._data[_name] = {
'data': full_data,
'meta': {
'x-static-large-object': 'true',
'x-object-sysmeta-slo-etag': etag,
'x-object-sysmeta-slo-size': str(len(full_data))
}
}
self._slo.append(_name)
data = json.dumps(res)
self.conn.object_create(self.account,
self._cnt,
obj_name=_name,
data=data)
self.conn.object_update(self.account, self._cnt, _name,
self._data[_name]['meta'])
def _check_tar(self, data):
raw = BytesIO(data)
tar = tarfile.open(fileobj=raw, ignore_zeros=True)
info = self._data.keys()
for entry in tar.getnames():
if entry == CONTAINER_MANIFEST:
# skip special entry
continue
self.assertIn(entry, info)
tmp = tar.extractfile(entry)
self.assertEqual(self._data[entry]['data'], tmp.read())
info.remove(entry)
self.assertEqual(info, [])
return tar
def _check_container(self, cnt):
ret = self.conn.object_list(account=self.account, container=cnt)
names = self._data.keys()
for obj in ret['objects']:
name = obj['name']
self.assertIn(name, self._data)
self.assertEqual(obj['size'], len(self._data[name]['data']))
_, data = self.conn.object_fetch(self.account, cnt, name)
raw = "".join(data)
self.assertEqual(
md5(raw).hexdigest(),
md5(self._data[name]['data']).hexdigest())
meta = self.conn.object_get_properties(self.account, cnt, name)
self.assertEqual(meta['properties'], self._data[name]['meta'])
names.remove(name)
self.assertEqual(len(names), 0)
def _simple_download(self,
name=gen_names,
metadata=None,
size=513,
append=False):
self._create_data(name=name,
metadata=metadata,
size=size,
append=append)
ret = requests.get(self._uri)
self.assertGreater(len(ret.content), 0)
self.assertEqual(ret.status_code, 200)
self.raw = ret.content
return self._check_tar(ret.content)
def _check_metadata(self, tar):
for entry in tar.getnames():
if entry == CONTAINER_MANIFEST:
# skip special entry
continue
headers = tar.getmember(entry).pax_headers
keys = headers.keys()[:]
for key, val in self._data[entry]['meta'].items():
key = u"SCHILY.xattr.user." + key.decode('utf-8')
self.assertIn(key, headers)
self.assertEqual(val.decode('utf-8'), headers[key])
keys.remove(key)
#
self.assertEqual(self._data[entry]['mime'], headers['mime_type'])
keys.remove('mime_type')
#
self.assertEqual(keys, [])
def test_missing_container(self):
ret = requests.get(self._streaming + '/' + random_container("ms-"))
self.assertEqual(ret.status_code, 404)
def test_invalid_url(self):
ret = requests.get(self._streaming)
self.assertEqual(ret.status_code, 404)
ret = requests.head(self._streaming + '/' + random_container('inv') +
'/' + random_container('inv'))
self.assertEqual(ret.status_code, 404)
def test_download_empty_container(self):
ret = requests.get(self._uri)
self.assertEqual(ret.status_code, 204)
def test_simple_download(self):
self._simple_download()
def test_check_head(self):
self._create_data()
get = requests.get(self._uri)
head = requests.head(self._uri)
self.assertEqual(get.headers['content-length'],
head.headers['content-length'])
def test_download_per_range(self):
self._create_data()
org = requests.get(self._uri)
data = []
for idx in xrange(0, int(org.headers['content-length']), 512):
ret = requests.get(
self._uri, headers={'Range': 'bytes=%d-%d' % (idx, idx + 511)})
self.assertEqual(ret.status_code, 206)
self.assertEqual(len(ret.content), 512)
self.assertEqual(ret.content, org.content[idx:idx + 512])
data.append(ret.content)
data = "".join(data)
self.assertGreater(len(data), 0)
self.assertEqual(md5(data).hexdigest(), md5(org.content).hexdigest())
def test_invalid_range(self):
self._create_data()
ranges = ((-512, 511), (512, 0), (1, 3), (98888, 99999))
for start, end in ranges:
ret = requests.get(self._uri,
headers={'Range': 'bytes=%d-%d' % (start, end)})
self.assertEqual(
ret.status_code, 416,
"Invalid error code for range %d-%d" % (start, end))
ret = requests.get(self._uri,
headers={'Range': 'bytes=0-511, 512-1023'})
self.assertEqual(ret.status_code, 416)
def test_file_metadata(self):
tar = self._simple_download(metadata=gen_metadata)
self._check_metadata(tar)
def test_container_metadata(self):
key, val = gen_metadata()
ret = self.conn.container_update(self.account, self._cnt, {key: val})
ret = self.conn.container_show(self.account, self._cnt)
ret = requests.get(self._uri)
self.assertEqual(ret.status_code, 200)
raw = BytesIO(ret.content)
tar = tarfile.open(fileobj=raw, ignore_zeros=True)
self.assertIn(CONTAINER_PROPERTIES, tar.getnames())
data = json.load(tar.extractfile(CONTAINER_PROPERTIES))
self.assertIn(key, data)
self.assertEqual(val, data[key])
def test_charset_file(self):
self._simple_download(name=gen_charset_names)
@unittest.skip("wip")
def test_byte_metadata(self):
tar = self._simple_download(metadata=gen_byte_metadata)
self._check_metadata(tar)
def test_charset_metadata(self):
tar = self._simple_download(metadata=gen_charset_metadata)
self._check_metadata(tar)
@attr('s3')
def test_s3_simple_download(self):
self._create_s3_slo()
ret = requests.get(self._uri)
self.assertGreater(len(ret.content), 0)
self.assertEqual(ret.status_code, 200)
self.raw = ret.content
raw = BytesIO(ret.content)
tar = tarfile.open(fileobj=raw, ignore_zeros=True)
info = self._data.keys()
for entry in tar.getnames():
if entry == CONTAINER_MANIFEST:
# skip special entry
continue
self.assertIn(entry, info)
tmp = tar.extractfile(entry)
self.assertEqual(self._data[entry]['data'], tmp.read())
info.remove(entry)
self.assertEqual(len(info), 0)
return tar
@attr('s3')
def test_s3_range_download(self):
self._create_s3_slo()
org = requests.get(self._uri)
self.assertEqual(org.status_code, 200)
data = []
for idx in xrange(0, int(org.headers['content-length']), 512):
ret = requests.get(
self._uri, headers={'Range': 'bytes=%d-%d' % (idx, idx + 511)})
self.assertEqual(ret.status_code, 206)
self.assertEqual(len(ret.content), 512)
self.assertEqual(ret.content, org.content[idx:idx + 512])
data.append(ret.content)
data = "".join(data)
self.assertGreater(len(data), 0)
self.assertEqual(md5(data).hexdigest(), md5(org.content).hexdigest())
@attr('s3')
def test_s3_check_slo_metadata_download(self):
self._create_s3_slo()
org = requests.get(self.make_uri('dump'))
self.assertEqual(org.status_code, 200)
cnt = rand_str(20)
res = requests.put(self.make_uri('restore', container=cnt),
data=org.content)
self.assertEqual(org.status_code, 200)
res = self.conn.object_get_properties(self.account, cnt, self._slo[0])
props = res['properties']
self.assertNotIn('x-static-large-object', props)
self.assertNotIn('x-object-sysmeta-slo-size', props)
self.assertNotIn('x-object-sysmeta-slo-etag', props)
@attr('simple')
def test_simple_restore(self):
self._create_data(metadata=gen_metadata)
org = requests.get(self.make_uri('dump'))
cnt = rand_str(20)
res = requests.put(self.make_uri('restore', container=cnt),
data=org.content)
self.assertEqual(res.status_code, 201)
self._check_container(cnt)
@attr('restore')
def test_multipart_restore(self):
self._create_data(metadata=gen_metadata, size=1025 * 1024)
org = requests.get(self.make_uri('dump'))
cnt = rand_str(20)
size = 1014 * 1024
parts = [
org.content[x:x + size] for x in xrange(0, len(org.content), size)
]
uri = self.make_uri('restore', container=cnt)
start = 0
for part in parts:
hdrs = {'Range': 'bytes=%d-%d' % (start, start + len(part) - 1)}
res = requests.put(uri, data=part, headers=hdrs)
start += len(part)
self.assertIn(res.status_code, [201, 206])
self._check_container(cnt)
@attr('restore')
def test_multipart_invalid_restore(self):
self._create_data(metadata=gen_metadata, size=1025 * 1024)
org = requests.get(self.make_uri('dump'))
cnt = rand_str(20)
uri = self.make_uri('restore', container=cnt)
size = 1014 * 1024
parts = [
org.content[x:x + size] for x in xrange(0, len(org.content), size)
]
start = 0
for part in parts:
hdrs = {'Range': 'bytes=%d-%d' % (start, start + len(part) - 1)}
res = requests.put(uri, data=part, headers=hdrs)
self.assertIn(res.status_code, [201, 206])
start += len(part)
# only unfinished restoration expose X-Consumed-Size
if res.status_code == 206:
res = requests.head(uri)
self.assertEqual(int(res.headers['X-Consumed-Size']), start)
inv = requests.put(uri, data=part, headers=hdrs)
self.assertEqual(inv.status_code, 422)
if res.status_code == 206:
res = requests.head(uri)
self.assertEqual(int(res.headers['X-Consumed-Size']), start)
uri = self.make_uri('restore', container=rand_str(20))
hdrs = {'Range': 'bytes=%d-%d' % (size, size + len(parts[1]) - 1)}
res = requests.put(uri, data=part, headers=hdrs)
self.assertEqual(res.status_code, 422)
self._check_container(cnt)
@attr('concurrency')
def test_multipart_concurrency(self):
self._create_data(metadata=gen_metadata, size=1025 * 1024)
org = requests.get(self.make_uri('dump'))
cnt = rand_str(20)
uri = self.make_uri('restore', container=cnt)
size = divmod(len(org.content) / 3, 512)[0] * 512
parts = [
org.content[x:x + size] for x in xrange(0, len(org.content), size)
]
start = 0
class StreamWithContentLength(Thread):
"""Thread to send data with delays to restore API"""
def __init__(self, data, headers):
self._count = 0
self._data = data
self._hdrs = headers
super(StreamWithContentLength, self).__init__()
def __len__(self):
return len(self._data)
def read(self, *args):
if self._count < len(self._data):
time.sleep(0.5)
data = self._data[self._count:self._count + size / 3]
self._count += len(data)
return data
return ""
def run(self):
self._ret = requests.put(uri, data=self, headers=self._hdrs)
for idx, part in enumerate(parts):
hdrs = {'Range': 'bytes=%d-%d' % (start, start + len(part) - 1)}
if idx == 0:
res = requests.put(uri, data=part, headers=hdrs)
self.assertIn(res.status_code, [201, 206])
else:
# launch Thread and simulate slow bandwidth
thr = StreamWithContentLength(part, hdrs)
thr.start()
# send data on same range
time.sleep(0.5)
res = requests.put(uri, data=part, headers=hdrs)
self.assertEqual(res.status_code, 422)
thr.join()
self.assertIn(thr._ret.status_code, [201, 206])
start += len(part)
self._check_container(cnt)
@attr('disconnected')
def test_broken_connectivity(self):
self._create_data(metadata=gen_metadata, size=1025 * 1024)
org = requests.get(self.make_uri('dump'))
cnt = rand_str(20)
class FakeStream(object):
"""Send data and simulate a connectivity issue"""
def __init__(self, data, size):
self._count = 0
self._data = data
self._size = size
def __len__(self):
return len(self._data)
def read(self, *args):
if self._count < self._size:
data = self._data[self._count:self._count + size / 3]
self._count += len(data)
return data
if self._count == len(self._data):
return ""
raise Exception("break connection")
def wait_lock():
"""When the lock is gone, return current consumed size"""
nb = 0
while True:
time.sleep(0.1)
req = requests.head(uri)
if (req.status_code == 200 and req.headers.get(
'X-Upload-In-Progress', '1') == '0'):
print("Tried before lock free", nb)
print("Got consumed-size", req.headers['X-Consumed-Size'])
return int(req.headers['X-Consumed-Size'])
nb += 1
self.assertLess(nb, 10)
uri = self.make_uri('restore', container=cnt)
block = 1000 * 512
start = 0
cut = False
while True:
if start:
start = wait_lock()
stop = min(len(org.content), start + block)
hdrs = {'Range': 'bytes=%d-%d' % (start, stop - 1)}
size = stop - start
if cut:
size = block / 2
cut = not cut
try:
ret = requests.put(uri,
headers=hdrs,
data=FakeStream(org.content[start:stop],
size))
except Exception:
pass
else:
self.assertIn(
ret.status_code, (201, 206),
"Unexpected %d HTTP response: %s" %
(ret.status_code, ret.content))
start += size
if ret.status_code == 201:
break
result = requests.get(self.make_uri('dump', container=cnt))
self._check_tar(result.content)
@attr('rawtar')
def test_rawtar(self):
"""Create a normal tar archive and restore it"""
raw = BytesIO()
tarfile = TarFile(mode='w', fileobj=raw)
testdata = rand_str(20) * 5000
inf = TarInfo("simpletar")
fileraw = BytesIO()
fileraw.write(testdata)
inf.size = len(testdata)
fileraw.seek(0)
tarfile.addfile(inf, fileobj=fileraw)
tarfile.close()
raw.seek(0)
data = raw.read()
cnt = rand_str(20)
ret = requests.put(self.make_uri("restore", container=cnt), data=data)
self.assertEqual(ret.status_code, 201)
meta, stream = self.conn.object_fetch(self.account, cnt, "simpletar")
self.assertEqual(
md5("".join(stream)).hexdigest(),
md5(testdata).hexdigest())
@attr('invalid')
def test_checksums(self):
"""Check restore operation with invalid tar"""
tar = self._simple_download(append=True)
manifest = json.load(tar.extractfile(CONTAINER_MANIFEST),
object_pairs_hook=OrderedDict)
# => add random bytes inside each file (either header and data)
for entry in manifest:
if entry['name'] == CONTAINER_MANIFEST:
# CONTAINER_MANIFEST does not have checksum at this time
continue
inv = self.raw
# Test with tar entry
# checksum tar doesn't work very well with SCHILY attributes
# so only apply changes on regular block entry
idx = entry['start_block'] * BLOCKSIZE \
+ random.randint(0, BLOCKSIZE)
# + random.randint(0, entry['hdr_blocks'] * BLOCKSIZE)
while self.raw[idx] == inv[idx]:
inv = inv[:idx] + chr(random.randint(0, 255)) + inv[idx + 1:]
cnt = rand_str(20)
res = requests.put(self.make_uri('restore', container=cnt),
data=inv)
self.assertEqual(res.status_code, 400)
# skip emty file
if entry['size'] == 0:
continue
# Test with data blocks
inv = self.raw
idx = (entry['start_block'] + entry['hdr_blocks']) * BLOCKSIZE \
+ random.randint(0, entry['size'] - 1)
while self.raw[idx] == inv[idx]:
inv = inv[:idx] + chr(random.randint(0, 255)) + inv[idx + 1:]
cnt = rand_str(20)
res = requests.put(self.make_uri('restore', container=cnt),
data=inv)
self.assertEqual(res.status_code, 400)
class TestObjectStorageAPI(BaseTestCase):
def setUp(self):
super(TestObjectStorageAPI, self).setUp()
self.api = ObjectStorageApi(self.ns, endpoint=self.uri)
self.created = list()
def tearDown(self):
super(TestObjectStorageAPI, self).tearDown()
for ct, name in self.created:
try:
self.api.object_delete(self.account, ct, name)
except Exception:
logging.exception("Failed to delete %s/%s/%s//%s", self.ns,
self.account, ct, name)
def _create(self, name, metadata=None):
return self.api.container_create(self.account,
name,
properties=metadata)
def _delete(self, name):
self.api.container_delete(self.account, name)
def _clean(self, name, clear=False):
if clear:
# must clean properties before
self.api.container_del_properties(self.account, name, [])
self._delete(name)
def _get_properties(self, name, properties=None):
return self.api.container_get_properties(self.account,
name,
properties=properties)
def _set_properties(self, name, properties=None):
return self.api.container_set_properties(self.account,
name,
properties=properties)
def test_container_show(self):
# container_show on unknown container
name = random_str(32)
self.assertRaises(exc.NoSuchContainer, self.api.container_show,
self.account, name)
self._create(name)
# container_show on existing container
res = self.api.container_show(self.account, name)
self.assertIsNot(res['properties'], None)
self._delete(name)
# container_show on deleted container
self.assertRaises(exc.NoSuchContainer, self.api.container_show,
self.account, name)
def test_container_create(self):
name = random_str(32)
res = self._create(name)
self.assertEqual(res, True)
# second create
res = self._create(name)
self.assertEqual(res, False)
# clean
self._delete(name)
def test_create_properties(self):
name = random_str(32)
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
res = self._create(name, metadata)
self.assertEqual(res, True)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
def test_container_delete(self):
name = random_str(32)
# container_delete on unknown container
self.assertRaises(exc.NoSuchContainer, self.api.container_delete,
self.account, name)
res = self._create(name)
self.assertEqual(res, True)
# container_delete on existing container
self._delete(name)
# verify deleted
self.assertRaises(exc.NoSuchContainer, self.api.container_show,
self.account, name)
# second delete
self.assertRaises(exc.NoSuchContainer, self.api.container_delete,
self.account, name)
# verify deleted
self.assertRaises(exc.NoSuchContainer, self.api.container_show,
self.account, name)
def test_container_get_properties(self):
name = random_str(32)
# container_get_properties on unknown container
self.assertRaises(exc.NoSuchContainer,
self.api.container_get_properties, self.account,
name)
res = self._create(name)
self.assertEqual(res, True)
# container_get_properties on existing container
data = self.api.container_get_properties(self.account, name)
self.assertEqual(data['properties'], {})
self.assertIsNot(data['system'], None)
self.assertIn("sys.user.name", data['system'])
# container_get_properties
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
self._set_properties(name, metadata)
data = self.api.container_get_properties(self.account, name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
# container_get_properties on deleted container
self.assertRaises(exc.NoSuchContainer,
self.api.container_get_properties, self.account,
name)
def test_container_get_properties_filtered(self):
self.skipTest("Server side properties filtering not implemented")
name = random_str(32)
res = self._create(name)
self.assertEqual(res, True)
# container_get_properties on existing container
data = self.api.container_get_properties(self.account, name)
self.assertEqual(data['properties'], {})
# container_get_properties
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
self._set_properties(name, metadata)
# container_get_properties specify key
key = metadata.keys().pop(0)
data = self.api.container_get_properties(self.account, name, [key])
self.assertEqual({key: metadata[key]}, data['properties'])
# clean
self._clean(name, True)
def test_container_set_properties(self):
name = random_str(32)
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
# container_set_properties on unknown container
self.assertRaises(exc.NoSuchContainer,
self.api.container_set_properties, self.account,
name, metadata)
res = self._create(name)
self.assertEqual(res, True)
# container_set_properties on existing container
self.api.container_set_properties(self.account, name, metadata)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# container_set_properties
key = random_str(32)
value = random_str(32)
metadata2 = {key: value}
self._set_properties(name, metadata2)
metadata.update(metadata2)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# container_set_properties overwrite key
key = metadata.keys().pop(0)
value = random_str(32)
metadata3 = {key: value}
metadata.update(metadata3)
self.api.container_set_properties(self.account, name, metadata3)
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
# container_set_properties on deleted container
self.assertRaises(exc.NoSuchContainer,
self.api.container_set_properties, self.account,
name, metadata)
def test_del_properties(self):
name = random_str(32)
metadata = {
random_str(32): random_str(32),
random_str(32): random_str(32),
}
# container_del_properties on unknown container
self.assertRaises(exc.NoSuchContainer,
self.api.container_del_properties, self.account,
name, [])
res = self._create(name, metadata)
self.assertEqual(res, True)
key = metadata.keys().pop()
del metadata[key]
# container_del_properties on existing container
self.api.container_del_properties(self.account, name, [key])
data = self._get_properties(name)
self.assertNotIn(key, data['properties'])
key = random_str(32)
# We do not check if a property exists before deleting it
# self.assertRaises(
# exc.NoSuchContainer, self.api.container_del_properties,
# self.account, name, [key])
self.api.container_del_properties(self.account, name, [key])
data = self._get_properties(name)
self.assertEqual(data['properties'], metadata)
# clean
self._clean(name, True)
# container_del_properties on deleted container
self.assertRaises(exc.NoSuchContainer,
self.api.container_del_properties, self.account,
name, metadata.keys())
def test_object_create_mime_type(self):
name = random_str(32)
self.api.object_create(self.account,
name,
data="data",
obj_name=name,
mime_type='text/custom')
meta, _ = self.api.object_locate(self.account, name, name)
self.assertEqual(meta['mime_type'], 'text/custom')
def _upload_data(self, name):
chunksize = int(self.conf["chunk_size"])
size = int(chunksize * 12)
data = random_data(int(size))
self.api.object_create(self.account, name, obj_name=name, data=data)
self.created.append((name, name))
_, chunks = self.api.object_locate(self.account, name, name)
logging.debug("Chunks: %s", chunks)
return sort_chunks(chunks, False), data
def _fetch_range(self, name, range_):
if not isinstance(range_[0], tuple):
ranges = (range_, )
else:
ranges = range_
stream = self.api.object_fetch(self.account, name, name,
ranges=ranges)[1]
data = ""
for chunk in stream:
data += chunk
return data
def test_object_fetch_range_start(self):
"""From 0 to somewhere"""
name = random_str(16)
_, data = self._upload_data(name)
end = 666
fdata = self._fetch_range(name, (0, end))
self.assertEqual(len(fdata), end + 1)
self.assertEqual(fdata, data[0:end + 1])
def test_object_fetch_range_end(self):
"""From somewhere to end"""
name = random_str(16)
chunks, data = self._upload_data(name)
start = 666
last = max(chunks.keys())
end = chunks[last][0]['offset'] + chunks[last][0]['size']
fdata = self._fetch_range(name, (start, end))
self.assertEqual(len(fdata), len(data) - start)
self.assertEqual(fdata, data[start:])
def test_object_fetch_range_metachunk_start(self):
"""From the start of the second metachunk to somewhere"""
name = random_str(16)
chunks, data = self._upload_data(name)
start = chunks[1][0]['offset']
end = start + 666
fdata = self._fetch_range(name, (start, end))
self.assertEqual(len(fdata), end - start + 1)
self.assertEqual(fdata, data[start:end + 1])
def test_object_fetch_range_metachunk_end(self):
"""From somewhere to end of the first metachunk"""
name = random_str(16)
chunks, data = self._upload_data(name)
start = 666
end = chunks[0][0]['size'] - 1
fdata = self._fetch_range(name, (start, end))
self.assertEqual(len(fdata), end - start + 1)
self.assertEqual(fdata, data[start:end + 1])
def test_object_fetch_range_2_metachunks(self):
"""
From somewhere in the first metachunk
to somewhere in the second metachunk
"""
name = random_str(16)
chunks, data = self._upload_data(name)
start = 666
end = start + chunks[0][0]['size'] - 1
fdata = self._fetch_range(name, (start, end))
self.assertEqual(len(fdata), end - start + 1)
self.assertEqual(fdata, data[start:end + 1])
def test_object_fetch_several_ranges(self):
"""
Download several ranges at once.
"""
name = random_str(16)
chunks, data = self._upload_data(name)
start = 666
end = start + chunks[0][0]['size'] - 1
fdata = self._fetch_range(name, ((start, end), (end + 1, end + 2)))
self.assertEqual(len(fdata), end - start + 3)
self.assertEqual(fdata, data[start:end + 3])
# Notice that we download some bytes from the second metachunk
# before some from the first.
fdata = self._fetch_range(
name, ((chunks[0][0]['size'], chunks[0][0]['size'] + 2), (0, 1),
(1, 2), (4, 6)))
self.assertEqual(len(fdata), 10)
self.assertEqual(
fdata, data[chunks[0][0]['size']:chunks[0][0]['size'] + 3] +
data[0:2] + data[1:3] + data[4:7])
def test_object_create_then_append(self):
"""Create an object then append data"""
name = random_str(16)
self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name)
_, size, _ = self.api.object_create(self.account,
name,
data="2" * 128,
obj_name=name,
append=True)
self.assertEqual(size, 128)
_, data = self.api.object_fetch(self.account, name, name)
data = "".join(data)
self.assertEqual(len(data), 256)
self.assertEqual(data, "1" * 128 + "2" * 128)
def test_object_create_from_append(self):
"""Create an object with append operation"""
name = random_str(16)
self.api.container_create(self.account, name)
self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name,
append=True)
_, data = self.api.object_fetch(self.account, name, name)
data = "".join(data)
self.assertEqual(len(data), 128)
self.assertEqual(data, "1" * 128)
def test_container_object_create_from_append(self):
"""Try to create container and object with append operation"""
name = random_str(16)
_chunks, size, checksum = self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name,
append=True)
self.assertEqual(size, 128)
meta = self.api.object_get_properties(self.account, name, name)
self.assertEqual(meta.get('hash', "").lower(), checksum.lower())
def test_container_refresh(self):
account = random_str(32)
# container_refresh on unknown container
name = random_str(32)
self.assertRaises(exc.NoSuchContainer, self.api.container_refresh,
account, name)
self.api.container_create(account, name)
time.sleep(0.5) # ensure container event have been processed
# container_refresh on existing container
self.api.container_refresh(account, name)
time.sleep(0.5) # ensure container event have been processed
res = self.api.container_list(account, prefix=name)
name_container, nb_objects, nb_bytes, _ = res[0]
self.assertEqual(name_container, name)
self.assertEqual(nb_objects, 0)
self.assertEqual(nb_bytes, 0)
self.api.object_create(account, name, data="data", obj_name=name)
time.sleep(0.5) # ensure container event have been processed
# container_refresh on existing container with data
self.api.container_refresh(account, name)
time.sleep(0.5) # ensure container event have been processed
res = self.api.container_list(account, prefix=name)
name_container, nb_objects, nb_bytes, _ = res[0]
self.assertEqual(name_container, name)
self.assertEqual(nb_objects, 1)
self.assertEqual(nb_bytes, 4)
self.api.object_delete(account, name, name)
time.sleep(0.5) # ensure container event have been processed
self.api.container_delete(account, name)
time.sleep(0.5) # ensure container event have been processed
# container_refresh on deleted container
self.assertRaises(exc.NoSuchContainer, self.api.container_refresh,
account, name)
self.api.account_delete(account)
def test_container_refresh_user_not_found(self):
name = random_str(32)
self.api.account.container_update(name, name, {"mtime": time.time()})
self.api.container_refresh(name, name)
containers = self.api.container_list(name)
self.assertEqual(len(containers), 0)
self.api.account_delete(name)
def test_account_refresh(self):
# account_refresh on unknown account
account = random_str(32)
self.assertRaises(exc.NoSuchAccount, self.api.account_refresh, account)
# account_refresh on existing account
self.api.account_create(account)
self.api.account_refresh(account)
time.sleep(0.5) # ensure container event have been processed
res = self.api.account_show(account)
self.assertEqual(res["bytes"], 0)
self.assertEqual(res["objects"], 0)
self.assertEqual(res["containers"], 0)
name = random_str(32)
self.api.object_create(account, name, data="data", obj_name=name)
time.sleep(0.5) # ensure container event have been processed
self.api.account_refresh(account)
time.sleep(0.5) # ensure container event have been processed
res = self.api.account_show(account)
self.assertEqual(res["bytes"], 4)
self.assertEqual(res["objects"], 1)
self.assertEqual(res["containers"], 1)
self.api.object_delete(account, name, name)
time.sleep(0.5) # ensure container event have been processed
self.api.container_delete(account, name)
time.sleep(0.5) # ensure container event have been processed
self.api.account_delete(account)
# account_refresh on deleted account
self.assertRaises(exc.NoSuchAccount, self.api.account_refresh, account)
def test_all_accounts_refresh(self):
# clear accounts
accounts = self.api.account_list()
for account in accounts:
try:
self.api.account_flush(account)
self.api.account_delete(account)
except exc.NoSuchAccount: # account remove in the meantime
pass
# all_accounts_refresh with 0 account
self.api.all_accounts_refresh()
# all_accounts_refresh with 2 account
account1 = random_str(32)
self.api.account_create(account1)
account2 = random_str(32)
self.api.account_create(account2)
self.api.all_accounts_refresh()
res = self.api.account_show(account1)
self.assertEqual(res["bytes"], 0)
self.assertEqual(res["objects"], 0)
self.assertEqual(res["containers"], 0)
res = self.api.account_show(account2)
self.assertEqual(res["bytes"], 0)
self.assertEqual(res["objects"], 0)
self.assertEqual(res["containers"], 0)
self.api.account_delete(account1)
self.api.account_delete(account2)
def test_account_flush(self):
# account_flush on unknown account
account = random_str(32)
self.assertRaises(exc.NoSuchAccount, self.api.account_flush, account)
# account_flush on existing account
name1 = random_str(32)
self.api.container_create(account, name1)
name2 = random_str(32)
self.api.container_create(account, name2)
time.sleep(0.5) # ensure container event have been processed
self.api.account_flush(account)
containers = self.api.container_list(account)
self.assertEqual(len(containers), 0)
res = self.api.account_show(account)
self.assertEqual(res["bytes"], 0)
self.assertEqual(res["objects"], 0)
self.assertEqual(res["containers"], 0)
self.api.container_delete(account, name1)
self.api.container_delete(account, name2)
time.sleep(0.5) # ensure container event have been processed
self.api.account_delete(account)
# account_flush on deleted account
self.assertRaises(exc.NoSuchAccount, self.api.account_flush, account)
def test_object_create_then_truncate(self):
"""Create an object then truncate data"""
name = random_str(16)
self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name)
self.api.object_truncate(self.account, name, name, size=64)
_, data = self.api.object_fetch(self.account, name, name)
data = "".join(data)
self.assertEqual(len(data), 64)
self.assertEqual(data, "1" * 64)
def test_object_create_append_then_truncate(self):
"""Create an object, append data then truncate on chunk boundary"""
name = random_str(16)
self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name)
_, size, _ = self.api.object_create(self.account,
name,
data="2" * 128,
obj_name=name,
append=True)
self.assertEqual(size, 128)
self.api.object_truncate(self.account, name, name, size=128)
_, data = self.api.object_fetch(self.account, name, name)
data = "".join(data)
self.assertEqual(len(data), 128)
self.assertEqual(data, "1" * 128)
self.api.object_truncate(self.account, name, name, size=128)
def test_object_create_then_invalid_truncate(self):
"""Create an object, append data then try to truncate outside object
range"""
name = random_str(16)
self.api.object_create(self.account,
name,
data="1" * 128,
obj_name=name)
self.assertRaises(exc.OioException,
self.api.object_truncate,
self.account,
name,
name,
size=-1)
self.assertRaises(exc.OioException,
self.api.object_truncate,
self.account,
name,
name,
size=129)
def test_container_snapshot(self):
name = random_str(16)
self.api.container_create(self.account, name)
test_object = "test_object"
self.api.object_create(self.account,
name,
data="0" * 128,
obj_name=test_object)
# Snapshot cannot have same name and same account
self.assertRaises(exc.ClientException, self.api.container_snapshot,
self.account, name, self.account, name)
snapshot_name = random_str(16)
self.assertNotEqual(snapshot_name, name)
# Non existing snapshot should work
self.api.container_snapshot(self.account, name, self.account,
snapshot_name)
# Already taken snapshot name should failed
self.assertRaises(exc.ClientException, self.api.container_snapshot,
self.account, name, self.account, snapshot_name)
# Check Container Frozen so create should failed
self.assertRaises(exc.ServiceBusy,
self.api.object_create,
self.account,
snapshot_name,
data="1" * 128,
obj_name="should_not_be_created")
# fullpath is set on every chunk
chunk_list = self.api.object_locate(self.account, name, test_object)[1]
# check that every chunk is different from the target
snapshot_list = self.api.object_locate(self.account, snapshot_name,
test_object)[1]
for c, t in zip(chunk_list, snapshot_list):
self.assertNotEqual(c['url'], t['url'])
# check target can be used
self.api.object_create(self.account,
name,
data="0" * 128,
obj_name="should_be_created")
# Create and send copy of a object
url_list = [c['url'] for c in chunk_list]
copy_list = self.api._generate_copy(url_list)
# every chunks should have the fullpath
fullpath = self.api._generate_fullpath(self.account, snapshot_name,
'copy', 12456)
self.api._send_copy(url_list, copy_list, fullpath[0])
# check that every copy exists
pool_manager = get_pool_manager()
for c in copy_list:
r = pool_manager.request('HEAD', c)
self.assertEqual(r.status, 200)
self.assertIn(fullpath[0],
r.headers["X-oio-chunk-meta-full-path"].split(','))
# Snapshot on non existing container should failed
self.assertRaises(exc.NoSuchContainer, self.api.container_snapshot,
random_str(16), random_str(16), random_str(16),
random_str(16))
# Snapshot need to have a account
self.assertRaises(exc.ClientException, self.api.container_snapshot,
self.account, name, None, random_str(16))
# Snapshot need to have a name
self.assertRaises(exc.ClientException, self.api.container_snapshot,
self.account, name, random_str(16), None)
