def test_sort_chunks(self):
raw_chunks = [
chunk("AAAA", "0"), chunk("BBBB", "0"),
chunk("CCCC", "1"), chunk("DDDD", "1"),
chunk("EEEE", "2"), chunk("FFFF", "2"),
]
chunks = _sort_chunks(raw_chunks, False)
sorted_chunks = {
0: [extend(chunk("AAAA", "0"), {"offset": 0}),
extend(chunk("BBBB", "0"), {"offset": 0})],
1: [extend(chunk("CCCC", "1"), {"offset": 32}),
extend(chunk("DDDD", "1"), {"offset": 32})],
2: [extend(chunk("EEEE", "2"), {"offset": 64}),
extend(chunk("FFFF", "2"), {"offset": 64})]
}
self.assertEqual(chunks, sorted_chunks)
raw_chunks = [
chunk("AAAA", "0.0"), chunk("BBBB", "0.1"), chunk("CCCC", "0.2"),
chunk("DDDD", "1.0"), chunk("EEEE", "1.1"), chunk("FFFF", "1.2"),
]
chunks = _sort_chunks(raw_chunks, True)
sorted_chunks = {
0: [extend(chunk("AAAA", "0.0"), {"num": 0, "offset": 0}),
extend(chunk("BBBB", "0.1"), {"num": 1, "offset": 0}),
extend(chunk("CCCC", "0.2"), {"num": 2, "offset": 0})],
1: [extend(chunk("DDDD", "1.0"), {"num": 0, "offset": 32}),
extend(chunk("EEEE", "1.1"), {"num": 1, "offset": 32}),
extend(chunk("FFFF", "1.2"), {"num": 2, "offset": 32})]
}
self.assertEqual(chunks, sorted_chunks)
def fetch(self):
storage_method = STORAGE_METHODS.load(self.chunk_method)
chunks = _sort_chunks(self.chunks.raw(),
storage_method.ec,
logger=self.logger)
stream = fetch_stream(chunks, None, storage_method)
return stream
def create(self, stream, **kwargs):
storage_method = STORAGE_METHODS.load(self.chunk_method)
sysmeta = self._generate_sysmeta()
chunks = _sort_chunks(self.chunks.raw(), storage_method.ec)
# TODO deal with headers
headers = {}
handler = ReplicatedWriteHandler(
stream, sysmeta, chunks, storage_method, headers=headers)
final_chunks, bytes_transferred, content_checksum = handler.stream()
# TODO sanity checks
self.checksum = content_checksum.upper()
self._create_object(**kwargs)
return final_chunks, bytes_transferred, content_checksum
def create(self, stream, **kwargs):
sysmeta = self._generate_sysmeta()
chunks = _sort_chunks(self.chunks.raw(), self.storage_method.ec)
headers = {}
handler = ECWriteHandler(stream,
sysmeta,
chunks,
self.storage_method,
headers=headers)
final_chunks, bytes_transferred, content_checksum = handler.stream()
# TODO sanity checks
self.checksum = content_checksum
self._create_object(**kwargs)
return final_chunks, bytes_transferred, content_checksum
def object_fetch(self, account, container, obj, version=None, ranges=None,
key_file=None, **kwargs):
meta, raw_chunks = self.object_locate(
account, container, obj, version=version, **kwargs)
chunk_method = meta['chunk_method']
storage_method = STORAGE_METHODS.load(chunk_method)
chunks = _sort_chunks(raw_chunks, storage_method.ec)
meta['container_id'] = cid_from_name(account, container).upper()
meta['ns'] = self.namespace
self._patch_timeouts(kwargs)
if storage_method.ec:
stream = fetch_stream_ec(chunks, ranges, storage_method, **kwargs)
elif storage_method.backblaze:
stream = self._fetch_stream_backblaze(meta, chunks, ranges,
storage_method, key_file,
**kwargs)
else:
stream = fetch_stream(chunks, ranges, storage_method, **kwargs)
return meta, stream
def object_truncate(self, account, container, obj,
version=None, size=None, **kwargs):
"""
Truncate object at specified size. Only shrink is supported.
A download may occur if size is not on chunk boundaries.
:param account: name of the account in which the object is stored
:param container: name of the container in which the object is stored
:param obj: name of the object to query
:param version: version of the object to query
:param size: new size of object
"""
# code copied from object_fetch (should be factorized !)
meta, raw_chunks = self.object_locate(
account, container, obj, version=version, **kwargs)
chunk_method = meta['chunk_method']
storage_method = STORAGE_METHODS.load(chunk_method)
chunks = _sort_chunks(raw_chunks, storage_method.ec)
for pos in sorted(chunks.keys()):
chunk = chunks[pos][0]
if (size >= chunk['offset']
and size <= chunk['offset'] + chunk['size']):
break
else:
raise exc.OioException("No chunk found at position %d" % size)
if chunk['offset'] != size:
# retrieve partial chunk
ret = self.object_fetch(account, container, obj,
version=version,
ranges=[(chunk['offset'], size-1)])
# TODO implement a proper object_update
pos = int(chunk['pos'].split('.')[0])
self.object_create(account, container, obj_name=obj,
data=ret[1], meta_pos=pos,
content_id=meta['id'])
return self.container.content_truncate(account, container, obj,
version=version, size=size,
**kwargs)
def fetch(self):
chunks = _sort_chunks(self.chunks.raw(), self.storage_method.ec)
stream = fetch_stream_ec(chunks, None, self.storage_method)
return stream
