def chunk_copy(self,
from_url,
to_url,
chunk_id=None,
fullpath=None,
cid=None,
path=None,
version=None,
content_id=None,
**kwargs):
stream = None
# Check source headers only when new fullpath is not provided
kwargs['check_headers'] = not bool(fullpath)
try:
meta, stream = self.chunk_get(from_url, **kwargs)
meta['oio_version'] = OIO_VERSION
meta['chunk_id'] = chunk_id or to_url.split('/')[-1]
meta['full_path'] = fullpath or meta['full_path']
meta['container_id'] = cid or meta.get('container_id')
meta['content_path'] = path or meta.get('content_path')
# FIXME: the original keys are the good ones.
# ReplicatedMetachunkWriter should be modified to accept them.
meta['version'] = version or meta.get('content_version')
meta['id'] = content_id or meta.get('content_id')
meta['chunk_method'] = meta['content_chunkmethod']
meta['policy'] = meta['content_policy']
bytes_transferred, chunk_hash = self.chunk_put(
to_url, meta, stream, **kwargs)
finally:
if stream:
stream.close()
try:
expected_chunk_size = meta.get('chunk_size')
if expected_chunk_size is not None:
expected_chunk_size = int(expected_chunk_size)
if bytes_transferred != expected_chunk_size:
raise exc.ChunkException(
'Size isn\'t the same for the copied chunk')
expected_chunk_hash = meta.get('chunk_hash')
if expected_chunk_hash is not None:
expected_chunk_hash = expected_chunk_hash.upper()
chunk_hash = chunk_hash.upper()
if chunk_hash != expected_chunk_hash:
# Should never happen, the hash is checked by the rawx
raise exc.ChunkException(
'Hash isn\'t the same for the copied chunk')
except exc.ChunkException:
# rollback
self.chunk_delete(to_url, **kwargs)
raise
def __iter__(self):
parts_iter = self.get_iter()
if not parts_iter:
raise exc.ChunkException()
for part in parts_iter:
for data in part['iter']:
yield data
raise StopIteration
def chunk_link(self, target, link, fullpath, headers=None, **kwargs):
hdrs = headers.copy()
if link is None:
link = self._generate_fullchunk_copy(target, **kwargs)
hdrs['Destination'] = link
hdrs[CHUNK_HEADERS['full_path']] = fullpath
resp = self.http_pool.request('COPY', self.resolve_url(target),
headers=hdrs)
if resp.status != 201:
raise exc.ChunkException(resp.status)
return resp, link
def chunk_link(self, target, link, fullpath, headers=None,
write_timeout=None, **kwargs):
hdrs = headers.copy()
if link is None:
link = self._generate_fullchunk_copy(target, **kwargs)
hdrs['Destination'] = link
hdrs[CHUNK_HEADERS['full_path']] = fullpath
if write_timeout is not None:
kwargs['read_timeout'] = write_timeout
resp = self._request('COPY', target, headers=hdrs, **kwargs)
if resp.status != 201:
raise exc.ChunkException(resp.status)
return resp, link
def chunk_link(self,
target,
link,
fullpath,
headers=None,
write_timeout=None,
**kwargs):
hdrs = headers.copy()
if link is None:
link = self._generate_fullchunk_copy(target, **kwargs)
elif not link.startswith('http://'):
offset = target.rfind('/')
maxlen = len(target) - offset - 1
link = target[:offset + 1] + link[:maxlen]
hdrs['Destination'] = link
hdrs[CHUNK_HEADERS['full_path']] = fullpath
if write_timeout is not None:
kwargs['read_timeout'] = write_timeout
resp = self._request('COPY', target, headers=hdrs, **kwargs)
if resp.status != 201:
raise exc.ChunkException(resp.status)
return resp, link
def move_chunk(self,
chunk_id,
service_id=None,
check_quality=False,
dry_run=False,
max_attempts=3,
**kwargs):
"""
Move a chunk to another place. Optionally ensure that the
new place is an improvement over the current one.
"""
if isinstance(chunk_id, Chunk):
current_chunk = chunk_id
chunk_id = current_chunk.id
service_id = current_chunk.host
else:
candidates = self.chunks.filter(id=chunk_id)
if len(candidates) > 1:
if service_id is None:
raise exc.ChunkException(
"Several chunks with ID %s and no service ID" %
(chunk_id, ))
candidates = candidates.filter(host=service_id)
current_chunk = candidates.one()
if current_chunk is None or current_chunk not in self.chunks:
raise exc.OrphanChunk("Chunk not found in content")
if service_id:
other_chunks = self.chunks.filter(
metapos=current_chunk.metapos).exclude(host=service_id).all()
else:
other_chunks = self.chunks.filter(
metapos=current_chunk.metapos).exclude(id=chunk_id).all()
spare_urls, qualities = self._get_spare_chunk(
other_chunks, [current_chunk],
position=current_chunk.pos,
check_quality=check_quality,
max_attempts=max_attempts,
**kwargs)
# Sort chunks by score to try to copy with higher score.
# When scores are close together (e.g. [95, 94, 94, 93, 50]),
# don't always start with the highest element.
duplicate_chunks = self.chunks \
.filter(pos=current_chunk.pos) \
.sort(key=lambda chunk: _get_weighted_random_score(chunk.raw()),
reverse=True) \
.all()
if dry_run:
self.logger.info('Dry-run: would copy chunk from %s to %s',
duplicate_chunks[0].url, spare_urls[0])
else:
# To reduce the load on the rawx to decommission,
# use one of the rawx with a copy of the chunk to move.
for src in duplicate_chunks:
try:
self.logger.info('Copying chunk from %s to %s', src.url,
spare_urls[0])
# TODO(FVE): retry to copy (max_attempts times)
self.blob_client.chunk_copy(src.url,
spare_urls[0],
chunk_id=chunk_id,
fullpath=self.full_path,
cid=self.container_id,
path=self.path,
version=self.version,
content_id=self.content_id,
**kwargs)
break
except Exception as err:
self.logger.warn('Failed to copy chunk from %s to %s: %s',
src.url, spare_urls[0], err)
if len(duplicate_chunks) == 1:
raise
else:
raise UnrecoverableContent(
'No copy available of chunk to move')
self._update_spare_chunk(current_chunk, spare_urls[0])
try:
self.blob_client.chunk_delete(current_chunk.url, **kwargs)
except Exception as err:
self.logger.warn("Failed to delete chunk %s: %s",
current_chunk.url, err)
current_chunk.url = spare_urls[0]
current_chunk.quality = qualities[current_chunk.url]
return current_chunk.raw()
