def worker_container(): proxy = ObjectStorageApi(NS) while True: try: name = QUEUE.get(timeout=TIMEOUT) except eventlet.queue.Empty: break while True: if VERBOSE: print("Deleting", name) try: proxy.container_delete(ACCOUNT, name) COUNTERS.add(1, 0) break except Exception as ex: if "Election failed" in str(ex): # wait default Election wait delay ELECTIONS.add(1, 0) time.sleep(20) continue print("Container %s: %s" % (name, str(ex)), file=sys.stderr) break QUEUE.task_done()
class TestObjectStorageApiPerformance(BaseTestCase): def setUp(self): super(TestObjectStorageApiPerformance, self).setUp() self.api = ObjectStorageApi(self.ns, endpoint=self.uri, source_address=('127.0.0.8', 0)) self.created = list() self.containers = set() def tearDown(self): super(TestObjectStorageApiPerformance, self).tearDown() for ct, name in self.created: try: self.api.object_delete(self.account, ct, name) self.containers.add(ct) except Exception: logging.exception("Failed to delete %s/%s/%s//%s", self.ns, self.account, ct, name) for ct in self.containers: try: self.api.container_delete(self.account, ct) except Exception: logging.exception('Failed to delete %s/%s/%s', self.ns, self.account, ct) def test_object_create_32_md5_checksum(self): container = self.__class__.__name__ + random_str(8) for i in range(32): obj = "obj-%03d" % i self.api.object_create(self.account, container, obj_name=obj, data=obj, chunk_checksum_algo='md5') self.created.append((container, obj)) def test_object_create_32_no_checksum(self): container = self.__class__.__name__ + random_str(8) for i in range(32): obj = "obj-%03d" % i self.api.object_create(self.account, container, obj_name=obj, data=obj, chunk_checksum_algo=None) self.created.append((container, obj)) def test_object_list_empty_container(self): """ Ensure object listing of an empty container takes less than 35ms. """ container = self.__class__.__name__ + random_str(8) self.api.container_create(self.account, container) self.containers.add(container) for _ in range(8): start = monotonic_time() self.api.object_list(self.account, container) duration = monotonic_time() - start logging.info("Object list took %.6fs", duration) self.assertLess(duration, 0.035)
class ObjectStorageApiTestBase(BaseTestCase): def setUp(self): super(ObjectStorageApiTestBase, self).setUp() self.api = ObjectStorageApi(self.ns, endpoint=self.uri) self.created = list() def tearDown(self): super(ObjectStorageApiTestBase, self).tearDown() for ct, name in self.created: try: self.api.object_delete(self.account, ct, name) except Exception: logging.exception("Failed to delete %s/%s/%s//%s", self.ns, self.account, ct, name) def _create(self, name, metadata=None): return self.api.container_create(self.account, name, properties=metadata) def _delete(self, name): self.api.container_delete(self.account, name) def _clean(self, name, clear=False): if clear: # must clean properties before self.api.container_del_properties(self.account, name, []) self._delete(name) def _get_properties(self, name, properties=None): return self.api.container_get_properties(self.account, name, properties=properties) def _set_properties(self, name, properties=None): return self.api.container_set_properties(self.account, name, properties=properties) def _upload_empty(self, container, *objs, **kwargs): """Upload empty objects to `container`""" for obj in objs: self.api.object_create(self.account, container, obj_name=obj, data="", **kwargs) self.created.append((container, obj))
class TestObjectStorageApiPerformance(BaseTestCase): def setUp(self): super(TestObjectStorageApiPerformance, self).setUp() self.api = ObjectStorageApi(self.ns, endpoint=self.uri) self.created = list() def tearDown(self): super(TestObjectStorageApiPerformance, self).tearDown() containers = set() for ct, name in self.created: try: self.api.object_delete(self.account, ct, name) containers.add(ct) except Exception: logging.exception("Failed to delete %s/%s/%s//%s", self.ns, self.account, ct, name) for ct in containers: try: self.api.container_delete(self.account, ct) except Exception: logging.exception('Failed to delete %s/%s/%s', self.ns, self.account, ct) def test_object_create_32_md5_checksum(self): container = self.__class__.__name__ + random_str(8) for i in range(32): obj = "obj-%03d" % i self.api.object_create(self.account, container, obj_name=obj, data=obj, chunk_checksum_algo='md5') self.created.append((container, obj)) def test_object_create_32_no_checksum(self): container = self.__class__.__name__ + random_str(8) for i in range(32): obj = "obj-%03d" % i self.api.object_create(self.account, container, obj_name=obj, data=obj, chunk_checksum_algo=None) self.created.append((container, obj))
def main(): args = options() global ACCOUNT, PROXY ACCOUNT = args.account PROXY = ObjectStorageApi("OPENIO") args.path = args.path.rstrip('/') if '/' in args.path: bucket, path = args.path.split('/', 1) else: bucket = args.path path = "" containers = [] _bucket = container_hierarchy(bucket, path) # we don't use placeholders, we use prefix path as prefix for entry in full_list(prefix=container_hierarchy(bucket, path)): name, _files, _size, _ = entry if name != _bucket and not name.startswith(_bucket + '%2F'): continue if _files: items = PROXY.object_list(ACCOUNT, name) objs = [_item['name'] for _item in items['objects']] PROXY.object_delete_many(ACCOUNT, name, objs=objs) print("Deleting", len(objs), "objects") containers.append(name) print("We have to delete", len(containers), "containers") for container in containers: print("Deleting", container) PROXY.container_delete(ACCOUNT, container)
class TestContainerDownload(BaseTestCase): def setUp(self): super(TestContainerDownload, self).setUp() # FIXME: should we use direct API from BaseTestCase # or still container.client ? self.conn = ObjectStorageApi(self.ns) self._streaming = 'http://' + self.get_service_url('container')[2] self._cnt = random_container() self._uri = self.make_uri('dump') self._data = {} self.conn.container_create(self.account, self._cnt) self.raw = "" self._slo = [] def make_uri(self, action, account=None, container=None): account = account or self.account container = container or self._cnt return '%s/v1.0/container/%s?acct=%s&ref=%s' % ( self._streaming, action, account, container) def tearDown(self): for name in self._data: self.conn.object_delete(self.account, self._cnt, name) self.conn.container_delete(self.account, self._cnt) super(TestContainerDownload, self).tearDown() def _create_data(self, name=gen_names, metadata=None, size=513, append=False): for idx, _name in itertools.islice(name(), 5): mime = random.choice(MIMETYPE) if append and size > 0: data = gen_data(size / 2 * idx) entry = {'data': data, 'meta': None, 'mime': mime} self.conn.object_create(self.account, self._cnt, obj_name=_name, data=data, mime_type=mime) data = gen_data(size / 2 * idx) self.conn.object_create(self.account, self._cnt, obj_name=_name, data=data, mime_type=mime, append=True) entry['data'] += data else: data = gen_data(size * idx) entry = {'data': data, 'meta': None, 'mime': mime} self.conn.object_create(self.account, self._cnt, obj_name=_name, data=data, mime_type=mime) if metadata: entry['meta'] = {} for _ in xrange(10): key, val = metadata() entry['meta'][key] = val self.conn.object_update(self.account, self._cnt, _name, entry['meta']) self._data[_name] = entry def _create_s3_slo(self, name=gen_names, metadata=None): # create a fake S3 bucket with a SLO object chunksize = 10000 parts = 5 res = [] full_data = "" self.conn.container_create(self.account, self._cnt + '+segments') _name = "toto" etag = rand_str(50) part_number = 1 for size in [chunksize] * parts + [444]: data = gen_data(size) res.append({ 'bytes': size, 'content_type': 'application/octect-stream', 'hash': md5(data).hexdigest().upper(), 'last_modified': '2017-06-21T12:42:47.000000', 'name': '/%s+segments/%s/%s/%d' % (self._cnt, _name, etag, part_number) }) self.conn.object_create(self.account, "%s+segments" % self._cnt, obj_name='%s/%s/%d' % (_name, etag, part_number), data=data) full_data += data part_number += 1 self._data[_name] = { 'data': full_data, 'meta': { 'x-static-large-object': 'true', 'x-object-sysmeta-slo-etag': etag, 'x-object-sysmeta-slo-size': str(len(full_data)) } } self._slo.append(_name) data = json.dumps(res) self.conn.object_create(self.account, self._cnt, obj_name=_name, data=data) self.conn.object_update(self.account, self._cnt, _name, self._data[_name]['meta']) def _check_tar(self, data): raw = BytesIO(data) tar = tarfile.open(fileobj=raw, ignore_zeros=True) info = self._data.keys() for entry in tar.getnames(): if entry == CONTAINER_MANIFEST: # skip special entry continue self.assertIn(entry, info) tmp = tar.extractfile(entry) self.assertEqual(self._data[entry]['data'], tmp.read()) info.remove(entry) self.assertEqual(info, []) return tar def _check_container(self, cnt): ret = self.conn.object_list(account=self.account, container=cnt) names = self._data.keys() for obj in ret['objects']: name = obj['name'] self.assertIn(name, self._data) self.assertEqual(obj['size'], len(self._data[name]['data'])) _, data = self.conn.object_fetch(self.account, cnt, name) raw = "".join(data) self.assertEqual( md5(raw).hexdigest(), md5(self._data[name]['data']).hexdigest()) meta = self.conn.object_get_properties(self.account, cnt, name) self.assertEqual(meta['properties'], self._data[name]['meta']) names.remove(name) self.assertEqual(len(names), 0) def _simple_download(self, name=gen_names, metadata=None, size=513, append=False): self._create_data(name=name, metadata=metadata, size=size, append=append) ret = requests.get(self._uri) self.assertGreater(len(ret.content), 0) self.assertEqual(ret.status_code, 200) self.raw = ret.content return self._check_tar(ret.content) def _check_metadata(self, tar): for entry in tar.getnames(): if entry == CONTAINER_MANIFEST: # skip special entry continue headers = tar.getmember(entry).pax_headers keys = headers.keys()[:] for key, val in self._data[entry]['meta'].items(): key = u"SCHILY.xattr.user." + key.decode('utf-8') self.assertIn(key, headers) self.assertEqual(val.decode('utf-8'), headers[key]) keys.remove(key) # self.assertEqual(self._data[entry]['mime'], headers['mime_type']) keys.remove('mime_type') # self.assertEqual(keys, []) def test_missing_container(self): ret = requests.get(self._streaming + '/' + random_container("ms-")) self.assertEqual(ret.status_code, 404) def test_invalid_url(self): ret = requests.get(self._streaming) self.assertEqual(ret.status_code, 404) ret = requests.head(self._streaming + '/' + random_container('inv') + '/' + random_container('inv')) self.assertEqual(ret.status_code, 404) def test_download_empty_container(self): ret = requests.get(self._uri) self.assertEqual(ret.status_code, 204) def test_simple_download(self): self._simple_download() def test_check_head(self): self._create_data() get = requests.get(self._uri) head = requests.head(self._uri) self.assertEqual(get.headers['content-length'], head.headers['content-length']) def test_download_per_range(self): self._create_data() org = requests.get(self._uri) data = [] for idx in xrange(0, int(org.headers['content-length']), 512): ret = requests.get( self._uri, headers={'Range': 'bytes=%d-%d' % (idx, idx + 511)}) self.assertEqual(ret.status_code, 206) self.assertEqual(len(ret.content), 512) self.assertEqual(ret.content, org.content[idx:idx + 512]) data.append(ret.content) data = "".join(data) self.assertGreater(len(data), 0) self.assertEqual(md5(data).hexdigest(), md5(org.content).hexdigest()) def test_invalid_range(self): self._create_data() ranges = ((-512, 511), (512, 0), (1, 3), (98888, 99999)) for start, end in ranges: ret = requests.get(self._uri, headers={'Range': 'bytes=%d-%d' % (start, end)}) self.assertEqual( ret.status_code, 416, "Invalid error code for range %d-%d" % (start, end)) ret = requests.get(self._uri, headers={'Range': 'bytes=0-511, 512-1023'}) self.assertEqual(ret.status_code, 416) def test_file_metadata(self): tar = self._simple_download(metadata=gen_metadata) self._check_metadata(tar) def test_container_metadata(self): key, val = gen_metadata() ret = self.conn.container_update(self.account, self._cnt, {key: val}) ret = self.conn.container_show(self.account, self._cnt) ret = requests.get(self._uri) self.assertEqual(ret.status_code, 200) raw = BytesIO(ret.content) tar = tarfile.open(fileobj=raw, ignore_zeros=True) self.assertIn(CONTAINER_PROPERTIES, tar.getnames()) data = json.load(tar.extractfile(CONTAINER_PROPERTIES)) self.assertIn(key, data) self.assertEqual(val, data[key]) def test_charset_file(self): self._simple_download(name=gen_charset_names) @unittest.skip("wip") def test_byte_metadata(self): tar = self._simple_download(metadata=gen_byte_metadata) self._check_metadata(tar) def test_charset_metadata(self): tar = self._simple_download(metadata=gen_charset_metadata) self._check_metadata(tar) @attr('s3') def test_s3_simple_download(self): self._create_s3_slo() ret = requests.get(self._uri) self.assertGreater(len(ret.content), 0) self.assertEqual(ret.status_code, 200) self.raw = ret.content raw = BytesIO(ret.content) tar = tarfile.open(fileobj=raw, ignore_zeros=True) info = self._data.keys() for entry in tar.getnames(): if entry == CONTAINER_MANIFEST: # skip special entry continue self.assertIn(entry, info) tmp = tar.extractfile(entry) self.assertEqual(self._data[entry]['data'], tmp.read()) info.remove(entry) self.assertEqual(len(info), 0) return tar @attr('s3') def test_s3_range_download(self): self._create_s3_slo() org = requests.get(self._uri) self.assertEqual(org.status_code, 200) data = [] for idx in xrange(0, int(org.headers['content-length']), 512): ret = requests.get( self._uri, headers={'Range': 'bytes=%d-%d' % (idx, idx + 511)}) self.assertEqual(ret.status_code, 206) self.assertEqual(len(ret.content), 512) self.assertEqual(ret.content, org.content[idx:idx + 512]) data.append(ret.content) data = "".join(data) self.assertGreater(len(data), 0) self.assertEqual(md5(data).hexdigest(), md5(org.content).hexdigest()) @attr('s3') def test_s3_check_slo_metadata_download(self): self._create_s3_slo() org = requests.get(self.make_uri('dump')) self.assertEqual(org.status_code, 200) cnt = rand_str(20) res = requests.put(self.make_uri('restore', container=cnt), data=org.content) self.assertEqual(org.status_code, 200) res = self.conn.object_get_properties(self.account, cnt, self._slo[0]) props = res['properties'] self.assertNotIn('x-static-large-object', props) self.assertNotIn('x-object-sysmeta-slo-size', props) self.assertNotIn('x-object-sysmeta-slo-etag', props) @attr('simple') def test_simple_restore(self): self._create_data(metadata=gen_metadata) org = requests.get(self.make_uri('dump')) cnt = rand_str(20) res = requests.put(self.make_uri('restore', container=cnt), data=org.content) self.assertEqual(res.status_code, 201) self._check_container(cnt) @attr('restore') def test_multipart_restore(self): self._create_data(metadata=gen_metadata, size=1025 * 1024) org = requests.get(self.make_uri('dump')) cnt = rand_str(20) size = 1014 * 1024 parts = [ org.content[x:x + size] for x in xrange(0, len(org.content), size) ] uri = self.make_uri('restore', container=cnt) start = 0 for part in parts: hdrs = {'Range': 'bytes=%d-%d' % (start, start + len(part) - 1)} res = requests.put(uri, data=part, headers=hdrs) start += len(part) self.assertIn(res.status_code, [201, 206]) self._check_container(cnt) @attr('restore') def test_multipart_invalid_restore(self): self._create_data(metadata=gen_metadata, size=1025 * 1024) org = requests.get(self.make_uri('dump')) cnt = rand_str(20) uri = self.make_uri('restore', container=cnt) size = 1014 * 1024 parts = [ org.content[x:x + size] for x in xrange(0, len(org.content), size) ] start = 0 for part in parts: hdrs = {'Range': 'bytes=%d-%d' % (start, start + len(part) - 1)} res = requests.put(uri, data=part, headers=hdrs) self.assertIn(res.status_code, [201, 206]) start += len(part) # only unfinished restoration expose X-Consumed-Size if res.status_code == 206: res = requests.head(uri) self.assertEqual(int(res.headers['X-Consumed-Size']), start) inv = requests.put(uri, data=part, headers=hdrs) self.assertEqual(inv.status_code, 422) if res.status_code == 206: res = requests.head(uri) self.assertEqual(int(res.headers['X-Consumed-Size']), start) uri = self.make_uri('restore', container=rand_str(20)) hdrs = {'Range': 'bytes=%d-%d' % (size, size + len(parts[1]) - 1)} res = requests.put(uri, data=part, headers=hdrs) self.assertEqual(res.status_code, 422) self._check_container(cnt) @attr('concurrency') def test_multipart_concurrency(self): self._create_data(metadata=gen_metadata, size=1025 * 1024) org = requests.get(self.make_uri('dump')) cnt = rand_str(20) uri = self.make_uri('restore', container=cnt) size = divmod(len(org.content) / 3, 512)[0] * 512 parts = [ org.content[x:x + size] for x in xrange(0, len(org.content), size) ] start = 0 class StreamWithContentLength(Thread): """Thread to send data with delays to restore API""" def __init__(self, data, headers): self._count = 0 self._data = data self._hdrs = headers super(StreamWithContentLength, self).__init__() def __len__(self): return len(self._data) def read(self, *args): if self._count < len(self._data): time.sleep(0.5) data = self._data[self._count:self._count + size / 3] self._count += len(data) return data return "" def run(self): self._ret = requests.put(uri, data=self, headers=self._hdrs) for idx, part in enumerate(parts): hdrs = {'Range': 'bytes=%d-%d' % (start, start + len(part) - 1)} if idx == 0: res = requests.put(uri, data=part, headers=hdrs) self.assertIn(res.status_code, [201, 206]) else: # launch Thread and simulate slow bandwidth thr = StreamWithContentLength(part, hdrs) thr.start() # send data on same range time.sleep(0.5) res = requests.put(uri, data=part, headers=hdrs) self.assertEqual(res.status_code, 422) thr.join() self.assertIn(thr._ret.status_code, [201, 206]) start += len(part) self._check_container(cnt) @attr('disconnected') def test_broken_connectivity(self): self._create_data(metadata=gen_metadata, size=1025 * 1024) org = requests.get(self.make_uri('dump')) cnt = rand_str(20) class FakeStream(object): """Send data and simulate a connectivity issue""" def __init__(self, data, size): self._count = 0 self._data = data self._size = size def __len__(self): return len(self._data) def read(self, *args): if self._count < self._size: data = self._data[self._count:self._count + size / 3] self._count += len(data) return data if self._count == len(self._data): return "" raise Exception("break connection") def wait_lock(): """When the lock is gone, return current consumed size""" nb = 0 while True: time.sleep(0.1) req = requests.head(uri) if (req.status_code == 200 and req.headers.get( 'X-Upload-In-Progress', '1') == '0'): print("Tried before lock free", nb) print("Got consumed-size", req.headers['X-Consumed-Size']) return int(req.headers['X-Consumed-Size']) nb += 1 self.assertLess(nb, 10) uri = self.make_uri('restore', container=cnt) block = 1000 * 512 start = 0 cut = False while True: if start: start = wait_lock() stop = min(len(org.content), start + block) hdrs = {'Range': 'bytes=%d-%d' % (start, stop - 1)} size = stop - start if cut: size = block / 2 cut = not cut try: ret = requests.put(uri, headers=hdrs, data=FakeStream(org.content[start:stop], size)) except Exception: pass else: self.assertIn( ret.status_code, (201, 206), "Unexpected %d HTTP response: %s" % (ret.status_code, ret.content)) start += size if ret.status_code == 201: break result = requests.get(self.make_uri('dump', container=cnt)) self._check_tar(result.content) @attr('rawtar') def test_rawtar(self): """Create a normal tar archive and restore it""" raw = BytesIO() tarfile = TarFile(mode='w', fileobj=raw) testdata = rand_str(20) * 5000 inf = TarInfo("simpletar") fileraw = BytesIO() fileraw.write(testdata) inf.size = len(testdata) fileraw.seek(0) tarfile.addfile(inf, fileobj=fileraw) tarfile.close() raw.seek(0) data = raw.read() cnt = rand_str(20) ret = requests.put(self.make_uri("restore", container=cnt), data=data) self.assertEqual(ret.status_code, 201) meta, stream = self.conn.object_fetch(self.account, cnt, "simpletar") self.assertEqual( md5("".join(stream)).hexdigest(), md5(testdata).hexdigest()) @attr('invalid') def test_checksums(self): """Check restore operation with invalid tar""" tar = self._simple_download(append=True) manifest = json.load(tar.extractfile(CONTAINER_MANIFEST), object_pairs_hook=OrderedDict) # => add random bytes inside each file (either header and data) for entry in manifest: if entry['name'] == CONTAINER_MANIFEST: # CONTAINER_MANIFEST does not have checksum at this time continue inv = self.raw # Test with tar entry # checksum tar doesn't work very well with SCHILY attributes # so only apply changes on regular block entry idx = entry['start_block'] * BLOCKSIZE \ + random.randint(0, BLOCKSIZE) # + random.randint(0, entry['hdr_blocks'] * BLOCKSIZE) while self.raw[idx] == inv[idx]: inv = inv[:idx] + chr(random.randint(0, 255)) + inv[idx + 1:] cnt = rand_str(20) res = requests.put(self.make_uri('restore', container=cnt), data=inv) self.assertEqual(res.status_code, 400) # skip emty file if entry['size'] == 0: continue # Test with data blocks inv = self.raw idx = (entry['start_block'] + entry['hdr_blocks']) * BLOCKSIZE \ + random.randint(0, entry['size'] - 1) while self.raw[idx] == inv[idx]: inv = inv[:idx] + chr(random.randint(0, 255)) + inv[idx + 1:] cnt = rand_str(20) res = requests.put(self.make_uri('restore', container=cnt), data=inv) self.assertEqual(res.status_code, 400)
class TestObjectStorageAPI(BaseTestCase): def setUp(self): super(TestObjectStorageAPI, self).setUp() self.api = ObjectStorageApi(self.ns, endpoint=self.uri) self.created = list() def tearDown(self): super(TestObjectStorageAPI, self).tearDown() for ct, name in self.created: try: self.api.object_delete(self.account, ct, name) except Exception: logging.exception("Failed to delete %s/%s/%s//%s", self.ns, self.account, ct, name) def _create(self, name, metadata=None): return self.api.container_create(self.account, name, properties=metadata) def _delete(self, name): self.api.container_delete(self.account, name) def _clean(self, name, clear=False): if clear: # must clean properties before self.api.container_del_properties(self.account, name, []) self._delete(name) def _get_properties(self, name, properties=None): return self.api.container_get_properties(self.account, name, properties=properties) def _set_properties(self, name, properties=None): return self.api.container_set_properties(self.account, name, properties=properties) def test_container_show(self): # container_show on unknown container name = random_str(32) self.assertRaises(exc.NoSuchContainer, self.api.container_show, self.account, name) self._create(name) # container_show on existing container res = self.api.container_show(self.account, name) self.assertIsNot(res['properties'], None) self._delete(name) # container_show on deleted container self.assertRaises(exc.NoSuchContainer, self.api.container_show, self.account, name) def test_container_create(self): name = random_str(32) res = self._create(name) self.assertEqual(res, True) # second create res = self._create(name) self.assertEqual(res, False) # clean self._delete(name) def test_create_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } res = self._create(name, metadata) self.assertEqual(res, True) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) def test_container_delete(self): name = random_str(32) # container_delete on unknown container self.assertRaises(exc.NoSuchContainer, self.api.container_delete, self.account, name) res = self._create(name) self.assertEqual(res, True) # container_delete on existing container self._delete(name) # verify deleted self.assertRaises(exc.NoSuchContainer, self.api.container_show, self.account, name) # second delete self.assertRaises(exc.NoSuchContainer, self.api.container_delete, self.account, name) # verify deleted self.assertRaises(exc.NoSuchContainer, self.api.container_show, self.account, name) def test_container_get_properties(self): name = random_str(32) # container_get_properties on unknown container self.assertRaises(exc.NoSuchContainer, self.api.container_get_properties, self.account, name) res = self._create(name) self.assertEqual(res, True) # container_get_properties on existing container data = self.api.container_get_properties(self.account, name) self.assertEqual(data['properties'], {}) self.assertIsNot(data['system'], None) self.assertIn("sys.user.name", data['system']) # container_get_properties metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } self._set_properties(name, metadata) data = self.api.container_get_properties(self.account, name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) # container_get_properties on deleted container self.assertRaises(exc.NoSuchContainer, self.api.container_get_properties, self.account, name) def test_container_get_properties_filtered(self): self.skipTest("Server side properties filtering not implemented") name = random_str(32) res = self._create(name) self.assertEqual(res, True) # container_get_properties on existing container data = self.api.container_get_properties(self.account, name) self.assertEqual(data['properties'], {}) # container_get_properties metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } self._set_properties(name, metadata) # container_get_properties specify key key = metadata.keys().pop(0) data = self.api.container_get_properties(self.account, name, [key]) self.assertEqual({key: metadata[key]}, data['properties']) # clean self._clean(name, True) def test_container_set_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } # container_set_properties on unknown container self.assertRaises(exc.NoSuchContainer, self.api.container_set_properties, self.account, name, metadata) res = self._create(name) self.assertEqual(res, True) # container_set_properties on existing container self.api.container_set_properties(self.account, name, metadata) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # container_set_properties key = random_str(32) value = random_str(32) metadata2 = {key: value} self._set_properties(name, metadata2) metadata.update(metadata2) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # container_set_properties overwrite key key = metadata.keys().pop(0) value = random_str(32) metadata3 = {key: value} metadata.update(metadata3) self.api.container_set_properties(self.account, name, metadata3) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) # container_set_properties on deleted container self.assertRaises(exc.NoSuchContainer, self.api.container_set_properties, self.account, name, metadata) def test_del_properties(self): name = random_str(32) metadata = { random_str(32): random_str(32), random_str(32): random_str(32), } # container_del_properties on unknown container self.assertRaises(exc.NoSuchContainer, self.api.container_del_properties, self.account, name, []) res = self._create(name, metadata) self.assertEqual(res, True) key = metadata.keys().pop() del metadata[key] # container_del_properties on existing container self.api.container_del_properties(self.account, name, [key]) data = self._get_properties(name) self.assertNotIn(key, data['properties']) key = random_str(32) # We do not check if a property exists before deleting it # self.assertRaises( # exc.NoSuchContainer, self.api.container_del_properties, # self.account, name, [key]) self.api.container_del_properties(self.account, name, [key]) data = self._get_properties(name) self.assertEqual(data['properties'], metadata) # clean self._clean(name, True) # container_del_properties on deleted container self.assertRaises(exc.NoSuchContainer, self.api.container_del_properties, self.account, name, metadata.keys()) def test_object_create_mime_type(self): name = random_str(32) self.api.object_create(self.account, name, data="data", obj_name=name, mime_type='text/custom') meta, _ = self.api.object_locate(self.account, name, name) self.assertEqual(meta['mime_type'], 'text/custom') def _upload_data(self, name): chunksize = int(self.conf["chunk_size"]) size = int(chunksize * 12) data = random_data(int(size)) self.api.object_create(self.account, name, obj_name=name, data=data) self.created.append((name, name)) _, chunks = self.api.object_locate(self.account, name, name) logging.debug("Chunks: %s", chunks) return sort_chunks(chunks, False), data def _fetch_range(self, name, range_): if not isinstance(range_[0], tuple): ranges = (range_, ) else: ranges = range_ stream = self.api.object_fetch(self.account, name, name, ranges=ranges)[1] data = "" for chunk in stream: data += chunk return data def test_object_fetch_range_start(self): """From 0 to somewhere""" name = random_str(16) _, data = self._upload_data(name) end = 666 fdata = self._fetch_range(name, (0, end)) self.assertEqual(len(fdata), end + 1) self.assertEqual(fdata, data[0:end + 1]) def test_object_fetch_range_end(self): """From somewhere to end""" name = random_str(16) chunks, data = self._upload_data(name) start = 666 last = max(chunks.keys()) end = chunks[last][0]['offset'] + chunks[last][0]['size'] fdata = self._fetch_range(name, (start, end)) self.assertEqual(len(fdata), len(data) - start) self.assertEqual(fdata, data[start:]) def test_object_fetch_range_metachunk_start(self): """From the start of the second metachunk to somewhere""" name = random_str(16) chunks, data = self._upload_data(name) start = chunks[1][0]['offset'] end = start + 666 fdata = self._fetch_range(name, (start, end)) self.assertEqual(len(fdata), end - start + 1) self.assertEqual(fdata, data[start:end + 1]) def test_object_fetch_range_metachunk_end(self): """From somewhere to end of the first metachunk""" name = random_str(16) chunks, data = self._upload_data(name) start = 666 end = chunks[0][0]['size'] - 1 fdata = self._fetch_range(name, (start, end)) self.assertEqual(len(fdata), end - start + 1) self.assertEqual(fdata, data[start:end + 1]) def test_object_fetch_range_2_metachunks(self): """ From somewhere in the first metachunk to somewhere in the second metachunk """ name = random_str(16) chunks, data = self._upload_data(name) start = 666 end = start + chunks[0][0]['size'] - 1 fdata = self._fetch_range(name, (start, end)) self.assertEqual(len(fdata), end - start + 1) self.assertEqual(fdata, data[start:end + 1]) def test_object_fetch_several_ranges(self): """ Download several ranges at once. """ name = random_str(16) chunks, data = self._upload_data(name) start = 666 end = start + chunks[0][0]['size'] - 1 fdata = self._fetch_range(name, ((start, end), (end + 1, end + 2))) self.assertEqual(len(fdata), end - start + 3) self.assertEqual(fdata, data[start:end + 3]) # Notice that we download some bytes from the second metachunk # before some from the first. fdata = self._fetch_range( name, ((chunks[0][0]['size'], chunks[0][0]['size'] + 2), (0, 1), (1, 2), (4, 6))) self.assertEqual(len(fdata), 10) self.assertEqual( fdata, data[chunks[0][0]['size']:chunks[0][0]['size'] + 3] + data[0:2] + data[1:3] + data[4:7]) def test_object_create_then_append(self): """Create an object then append data""" name = random_str(16) self.api.object_create(self.account, name, data="1" * 128, obj_name=name) _, size, _ = self.api.object_create(self.account, name, data="2" * 128, obj_name=name, append=True) self.assertEqual(size, 128) _, data = self.api.object_fetch(self.account, name, name) data = "".join(data) self.assertEqual(len(data), 256) self.assertEqual(data, "1" * 128 + "2" * 128) def test_object_create_from_append(self): """Create an object with append operation""" name = random_str(16) self.api.container_create(self.account, name) self.api.object_create(self.account, name, data="1" * 128, obj_name=name, append=True) _, data = self.api.object_fetch(self.account, name, name) data = "".join(data) self.assertEqual(len(data), 128) self.assertEqual(data, "1" * 128) def test_container_object_create_from_append(self): """Try to create container and object with append operation""" name = random_str(16) _chunks, size, checksum = self.api.object_create(self.account, name, data="1" * 128, obj_name=name, append=True) self.assertEqual(size, 128) meta = self.api.object_get_properties(self.account, name, name) self.assertEqual(meta.get('hash', "").lower(), checksum.lower()) def test_container_refresh(self): account = random_str(32) # container_refresh on unknown container name = random_str(32) self.assertRaises(exc.NoSuchContainer, self.api.container_refresh, account, name) self.api.container_create(account, name) time.sleep(0.5) # ensure container event have been processed # container_refresh on existing container self.api.container_refresh(account, name) time.sleep(0.5) # ensure container event have been processed res = self.api.container_list(account, prefix=name) name_container, nb_objects, nb_bytes, _ = res[0] self.assertEqual(name_container, name) self.assertEqual(nb_objects, 0) self.assertEqual(nb_bytes, 0) self.api.object_create(account, name, data="data", obj_name=name) time.sleep(0.5) # ensure container event have been processed # container_refresh on existing container with data self.api.container_refresh(account, name) time.sleep(0.5) # ensure container event have been processed res = self.api.container_list(account, prefix=name) name_container, nb_objects, nb_bytes, _ = res[0] self.assertEqual(name_container, name) self.assertEqual(nb_objects, 1) self.assertEqual(nb_bytes, 4) self.api.object_delete(account, name, name) time.sleep(0.5) # ensure container event have been processed self.api.container_delete(account, name) time.sleep(0.5) # ensure container event have been processed # container_refresh on deleted container self.assertRaises(exc.NoSuchContainer, self.api.container_refresh, account, name) self.api.account_delete(account) def test_container_refresh_user_not_found(self): name = random_str(32) self.api.account.container_update(name, name, {"mtime": time.time()}) self.api.container_refresh(name, name) containers = self.api.container_list(name) self.assertEqual(len(containers), 0) self.api.account_delete(name) def test_account_refresh(self): # account_refresh on unknown account account = random_str(32) self.assertRaises(exc.NoSuchAccount, self.api.account_refresh, account) # account_refresh on existing account self.api.account_create(account) self.api.account_refresh(account) time.sleep(0.5) # ensure container event have been processed res = self.api.account_show(account) self.assertEqual(res["bytes"], 0) self.assertEqual(res["objects"], 0) self.assertEqual(res["containers"], 0) name = random_str(32) self.api.object_create(account, name, data="data", obj_name=name) time.sleep(0.5) # ensure container event have been processed self.api.account_refresh(account) time.sleep(0.5) # ensure container event have been processed res = self.api.account_show(account) self.assertEqual(res["bytes"], 4) self.assertEqual(res["objects"], 1) self.assertEqual(res["containers"], 1) self.api.object_delete(account, name, name) time.sleep(0.5) # ensure container event have been processed self.api.container_delete(account, name) time.sleep(0.5) # ensure container event have been processed self.api.account_delete(account) # account_refresh on deleted account self.assertRaises(exc.NoSuchAccount, self.api.account_refresh, account) def test_all_accounts_refresh(self): # clear accounts accounts = self.api.account_list() for account in accounts: try: self.api.account_flush(account) self.api.account_delete(account) except exc.NoSuchAccount: # account remove in the meantime pass # all_accounts_refresh with 0 account self.api.all_accounts_refresh() # all_accounts_refresh with 2 account account1 = random_str(32) self.api.account_create(account1) account2 = random_str(32) self.api.account_create(account2) self.api.all_accounts_refresh() res = self.api.account_show(account1) self.assertEqual(res["bytes"], 0) self.assertEqual(res["objects"], 0) self.assertEqual(res["containers"], 0) res = self.api.account_show(account2) self.assertEqual(res["bytes"], 0) self.assertEqual(res["objects"], 0) self.assertEqual(res["containers"], 0) self.api.account_delete(account1) self.api.account_delete(account2) def test_account_flush(self): # account_flush on unknown account account = random_str(32) self.assertRaises(exc.NoSuchAccount, self.api.account_flush, account) # account_flush on existing account name1 = random_str(32) self.api.container_create(account, name1) name2 = random_str(32) self.api.container_create(account, name2) time.sleep(0.5) # ensure container event have been processed self.api.account_flush(account) containers = self.api.container_list(account) self.assertEqual(len(containers), 0) res = self.api.account_show(account) self.assertEqual(res["bytes"], 0) self.assertEqual(res["objects"], 0) self.assertEqual(res["containers"], 0) self.api.container_delete(account, name1) self.api.container_delete(account, name2) time.sleep(0.5) # ensure container event have been processed self.api.account_delete(account) # account_flush on deleted account self.assertRaises(exc.NoSuchAccount, self.api.account_flush, account) def test_object_create_then_truncate(self): """Create an object then truncate data""" name = random_str(16) self.api.object_create(self.account, name, data="1" * 128, obj_name=name) self.api.object_truncate(self.account, name, name, size=64) _, data = self.api.object_fetch(self.account, name, name) data = "".join(data) self.assertEqual(len(data), 64) self.assertEqual(data, "1" * 64) def test_object_create_append_then_truncate(self): """Create an object, append data then truncate on chunk boundary""" name = random_str(16) self.api.object_create(self.account, name, data="1" * 128, obj_name=name) _, size, _ = self.api.object_create(self.account, name, data="2" * 128, obj_name=name, append=True) self.assertEqual(size, 128) self.api.object_truncate(self.account, name, name, size=128) _, data = self.api.object_fetch(self.account, name, name) data = "".join(data) self.assertEqual(len(data), 128) self.assertEqual(data, "1" * 128) self.api.object_truncate(self.account, name, name, size=128) def test_object_create_then_invalid_truncate(self): """Create an object, append data then try to truncate outside object range""" name = random_str(16) self.api.object_create(self.account, name, data="1" * 128, obj_name=name) self.assertRaises(exc.OioException, self.api.object_truncate, self.account, name, name, size=-1) self.assertRaises(exc.OioException, self.api.object_truncate, self.account, name, name, size=129) def test_container_snapshot(self): name = random_str(16) self.api.container_create(self.account, name) test_object = "test_object" self.api.object_create(self.account, name, data="0" * 128, obj_name=test_object) # Snapshot cannot have same name and same account self.assertRaises(exc.ClientException, self.api.container_snapshot, self.account, name, self.account, name) snapshot_name = random_str(16) self.assertNotEqual(snapshot_name, name) # Non existing snapshot should work self.api.container_snapshot(self.account, name, self.account, snapshot_name) # Already taken snapshot name should failed self.assertRaises(exc.ClientException, self.api.container_snapshot, self.account, name, self.account, snapshot_name) # Check Container Frozen so create should failed self.assertRaises(exc.ServiceBusy, self.api.object_create, self.account, snapshot_name, data="1" * 128, obj_name="should_not_be_created") # fullpath is set on every chunk chunk_list = self.api.object_locate(self.account, name, test_object)[1] # check that every chunk is different from the target snapshot_list = self.api.object_locate(self.account, snapshot_name, test_object)[1] for c, t in zip(chunk_list, snapshot_list): self.assertNotEqual(c['url'], t['url']) # check target can be used self.api.object_create(self.account, name, data="0" * 128, obj_name="should_be_created") # Create and send copy of a object url_list = [c['url'] for c in chunk_list] copy_list = self.api._generate_copy(url_list) # every chunks should have the fullpath fullpath = self.api._generate_fullpath(self.account, snapshot_name, 'copy', 12456) self.api._send_copy(url_list, copy_list, fullpath[0]) # check that every copy exists pool_manager = get_pool_manager() for c in copy_list: r = pool_manager.request('HEAD', c) self.assertEqual(r.status, 200) self.assertIn(fullpath[0], r.headers["X-oio-chunk-meta-full-path"].split(',')) # Snapshot on non existing container should failed self.assertRaises(exc.NoSuchContainer, self.api.container_snapshot, random_str(16), random_str(16), random_str(16), random_str(16)) # Snapshot need to have a account self.assertRaises(exc.ClientException, self.api.container_snapshot, self.account, name, None, random_str(16)) # Snapshot need to have a name self.assertRaises(exc.ClientException, self.api.container_snapshot, self.account, name, random_str(16), None)