def test_sort_chunks(self): raw_chunks = [ chunk("AAAA", "0"), chunk("BBBB", "0"), chunk("CCCC", "1"), chunk("DDDD", "1"), chunk("EEEE", "2"), chunk("FFFF", "2"), ] chunks = _sort_chunks(raw_chunks, False) sorted_chunks = { 0: [extend(chunk("AAAA", "0"), {"offset": 0}), extend(chunk("BBBB", "0"), {"offset": 0})], 1: [extend(chunk("CCCC", "1"), {"offset": 32}), extend(chunk("DDDD", "1"), {"offset": 32})], 2: [extend(chunk("EEEE", "2"), {"offset": 64}), extend(chunk("FFFF", "2"), {"offset": 64})] } self.assertEqual(chunks, sorted_chunks) raw_chunks = [ chunk("AAAA", "0.0"), chunk("BBBB", "0.1"), chunk("CCCC", "0.2"), chunk("DDDD", "1.0"), chunk("EEEE", "1.1"), chunk("FFFF", "1.2"), ] chunks = _sort_chunks(raw_chunks, True) sorted_chunks = { 0: [extend(chunk("AAAA", "0.0"), {"num": 0, "offset": 0}), extend(chunk("BBBB", "0.1"), {"num": 1, "offset": 0}), extend(chunk("CCCC", "0.2"), {"num": 2, "offset": 0})], 1: [extend(chunk("DDDD", "1.0"), {"num": 0, "offset": 32}), extend(chunk("EEEE", "1.1"), {"num": 1, "offset": 32}), extend(chunk("FFFF", "1.2"), {"num": 2, "offset": 32})] } self.assertEqual(chunks, sorted_chunks)
def fetch(self): storage_method = STORAGE_METHODS.load(self.chunk_method) chunks = _sort_chunks(self.chunks.raw(), storage_method.ec, logger=self.logger) stream = fetch_stream(chunks, None, storage_method) return stream
def create(self, stream, **kwargs): storage_method = STORAGE_METHODS.load(self.chunk_method) sysmeta = self._generate_sysmeta() chunks = _sort_chunks(self.chunks.raw(), storage_method.ec) # TODO deal with headers headers = {} handler = ReplicatedWriteHandler( stream, sysmeta, chunks, storage_method, headers=headers) final_chunks, bytes_transferred, content_checksum = handler.stream() # TODO sanity checks self.checksum = content_checksum.upper() self._create_object(**kwargs) return final_chunks, bytes_transferred, content_checksum
def create(self, stream, **kwargs): sysmeta = self._generate_sysmeta() chunks = _sort_chunks(self.chunks.raw(), self.storage_method.ec) headers = {} handler = ECWriteHandler(stream, sysmeta, chunks, self.storage_method, headers=headers) final_chunks, bytes_transferred, content_checksum = handler.stream() # TODO sanity checks self.checksum = content_checksum self._create_object(**kwargs) return final_chunks, bytes_transferred, content_checksum
def object_fetch(self, account, container, obj, version=None, ranges=None, key_file=None, **kwargs): meta, raw_chunks = self.object_locate( account, container, obj, version=version, **kwargs) chunk_method = meta['chunk_method'] storage_method = STORAGE_METHODS.load(chunk_method) chunks = _sort_chunks(raw_chunks, storage_method.ec) meta['container_id'] = cid_from_name(account, container).upper() meta['ns'] = self.namespace self._patch_timeouts(kwargs) if storage_method.ec: stream = fetch_stream_ec(chunks, ranges, storage_method, **kwargs) elif storage_method.backblaze: stream = self._fetch_stream_backblaze(meta, chunks, ranges, storage_method, key_file, **kwargs) else: stream = fetch_stream(chunks, ranges, storage_method, **kwargs) return meta, stream
def object_truncate(self, account, container, obj, version=None, size=None, **kwargs): """ Truncate object at specified size. Only shrink is supported. A download may occur if size is not on chunk boundaries. :param account: name of the account in which the object is stored :param container: name of the container in which the object is stored :param obj: name of the object to query :param version: version of the object to query :param size: new size of object """ # code copied from object_fetch (should be factorized !) meta, raw_chunks = self.object_locate( account, container, obj, version=version, **kwargs) chunk_method = meta['chunk_method'] storage_method = STORAGE_METHODS.load(chunk_method) chunks = _sort_chunks(raw_chunks, storage_method.ec) for pos in sorted(chunks.keys()): chunk = chunks[pos][0] if (size >= chunk['offset'] and size <= chunk['offset'] + chunk['size']): break else: raise exc.OioException("No chunk found at position %d" % size) if chunk['offset'] != size: # retrieve partial chunk ret = self.object_fetch(account, container, obj, version=version, ranges=[(chunk['offset'], size-1)]) # TODO implement a proper object_update pos = int(chunk['pos'].split('.')[0]) self.object_create(account, container, obj_name=obj, data=ret[1], meta_pos=pos, content_id=meta['id']) return self.container.content_truncate(account, container, obj, version=version, size=size, **kwargs)
def fetch(self): chunks = _sort_chunks(self.chunks.raw(), self.storage_method.ec) stream = fetch_stream_ec(chunks, None, self.storage_method) return stream