def rebuild(self,
container_id,
content_id,
chunk_id_or_pos,
rawx_id=None,
try_chunk_delete=False,
allow_same_rawx=True):
"""
Try to find the chunk in the metadata of the specified object,
then rebuild it.
"""
try:
content = self.content_factory.get(container_id, content_id)
except ContentNotFound:
raise OrphanChunk('Content not found: possible orphan chunk')
chunk_size = 0
chunk_pos = None
if len(chunk_id_or_pos) < 32:
chunk_pos = chunk_id_or_pos
chunk_id = None
metapos = int(chunk_pos.split('.', 1)[0])
chunk_size = content.chunks.filter(metapos=metapos).all()[0].size
else:
if '/' in chunk_id_or_pos:
chunk_id = chunk_id_or_pos.rsplit('/', 1)[-1]
else:
chunk_id = chunk_id_or_pos
chunk = content.chunks.filter(id=chunk_id).one()
if chunk is None:
raise OrphanChunk(
'Chunk not found in content: possible orphan chunk')
elif rawx_id and chunk.host != rawx_id:
raise ValueError('Chunk does not belong to this rawx')
chunk_size = chunk.size
content.rebuild_chunk(chunk_id,
allow_same_rawx=allow_same_rawx,
chunk_pos=chunk_pos)
if try_chunk_delete:
try:
content.blob_client.chunk_delete(chunk.url)
self.logger.info("Old chunk %s deleted", chunk.url)
except Exception as exc:
self.logger.warn('Failed to delete old chunk %s: %s',
chunk.url, exc)
# This call does not raise exception if chunk is not referenced
if chunk_id is not None:
try:
self.rdir_client.chunk_delete(chunk.host, container_id,
content_id, chunk_id)
except Exception as exc:
self.logger.warn(
'Failed to delete chunk entry (%s) from the rdir (%s): %s',
chunk_id, chunk.host, exc)
return chunk_size
def encode_fullpath(self, chunk_inode, chunk_id, account, container, path,
version, content_id):
# check if chunk exists and has the same inode
if not is_hexa(chunk_id) or len(chunk_id) != STRLEN_CHUNKID:
raise ValueError('chunk ID must be hexadecimal (%s)' %
STRLEN_CHUNKID)
try:
chunk_inode2 = os.stat(self._get_path(chunk_id)).st_ino
except OSError:
raise OrphanChunk('No such chunk: possible orphan chunk')
if chunk_inode2 != chunk_inode:
raise OrphanChunk('Not the same inode: possible orphan chunk')
# check fullpath and chunk ID
if isinstance(version, basestring):
try:
version = int(version)
except ValueError:
raise ValueError('version must be a number')
if version <= 0:
raise ValueError('version must be positive')
if not is_hexa(content_id):
raise ValueError('content ID must be hexadecimal')
fullpath = encode_fullpath(account, container, path, version,
content_id.upper())
return chunk_id.upper(), fullpath
def rebuild_chunk(self, chunk_id, allow_same_rawx=False, chunk_pos=None):
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None and chunk_pos is None:
raise OrphanChunk("Chunk not found in content")
elif current_chunk is None:
chunk = {"pos": chunk_pos, "url": ""}
current_chunk = Chunk(chunk)
chunks = self.chunks.filter(metapos=current_chunk.metapos)\
.exclude(id=chunk_id)
if chunk_id is None:
current_chunk.size = chunks[0].size
current_chunk.checksum = chunks[0].checksum
broken_list = list()
if not allow_same_rawx:
broken_list.append(current_chunk)
spare_url = self._get_spare_chunk(chunks.all(), broken_list)
handler = ECRebuildHandler(chunks.raw(), current_chunk.subpos,
self.storage_method)
new_chunk = {'pos': current_chunk.pos, 'url': spare_url[0]}
new_chunk = Chunk(new_chunk)
stream = handler.rebuild()
meta = {}
meta['chunk_id'] = new_chunk.id
meta['chunk_pos'] = current_chunk.pos
meta['container_id'] = self.container_id
# FIXME: should be 'content_chunkmethod' everywhere
# but sadly it isn't
meta['chunk_method'] = self.chunk_method
# FIXME: should be 'content_id' everywhere
# but sadly it isn't
meta['id'] = self.content_id
meta['content_path'] = self.path
# FIXME: should be 'content_policy' everywhere
# but sadly it isn't
meta['policy'] = self.policy
# FIXME: should be 'content_version' everywhere
# but sadly it isn't
meta['version'] = self.version
meta['metachunk_hash'] = current_chunk.checksum
meta['metachunk_size'] = current_chunk.size
meta['full_path'] = self.full_path
meta['oio_version'] = OIO_VERSION
self.blob_client.chunk_put(spare_url[0], meta, GeneratorIO(stream))
if chunk_id is None:
self._add_raw_chunk(current_chunk, spare_url[0])
else:
self._update_spare_chunk(current_chunk, spare_url[0])
def _get_chunk_id_and_fullpath(self,
chunk_inode,
chunk_pos,
content,
chunk_id=None):
content.container_id, content.account, content.container_name = \
self._save_container(content.container_id, content.account,
content.container_name)
content.container_id, content.path, content.version, \
content.content_id = self._save_content(
content.container_id, content.path, content.version,
content.content_id)
chunks = content.chunks.filter(host=self.volume_id)
if chunk_id:
chunks = chunks.filter(id=chunk_id)
chunk = chunks.filter(pos=chunk_pos).one()
if chunk is None:
raise OrphanChunk('Chunk not found in content:'
'possible orphan chunk')
chunk_id, new_fullpath = self.encode_fullpath(
chunk_inode, chunk.id, content.account, content.container_name,
content.path, content.version, content.content_id)
return chunk_id, new_fullpath
def rebuild_chunk(self, chunk_id):
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None:
raise OrphanChunk("Chunk not found in content")
chunks = self.chunks.filter(metapos=current_chunk.metapos)\
.exclude(id=chunk_id)
spare_url = self._get_spare_chunk(chunks.all(), [current_chunk])
handler = ECRebuildHandler(
chunks.raw(), current_chunk.subpos, self.storage_method)
new_chunk = {'pos': current_chunk.pos, 'url': spare_url[0]}
new_chunk = Chunk(new_chunk)
stream = handler.rebuild()
meta = {}
meta['chunk_id'] = new_chunk.id
meta['chunk_pos'] = current_chunk.pos
meta['container_id'] = self.container_id
meta['content_chunkmethod'] = self.chunk_method
meta['content_id'] = self.content_id
meta['content_path'] = self.path
meta['content_policy'] = self.stgpol
meta['content_version'] = self.version
meta['metachunk_hash'] = current_chunk.checksum
meta['metachunk_size'] = current_chunk.size
self.blob_client.chunk_put(spare_url[0], meta, stream)
self._update_spare_chunk(current_chunk, spare_url[0])
def get_chunk_id_and_fullpath(
self, chunk_inode, chunk_pos, container_id, path, version,
chunk_id=None, account=None, container=None, content_id=None):
if account is None or container is None:
account, container = self.name_from_cid(container_id)
if content_id:
try:
content = self.content_factory.get(
container_id, content_id,
account=account, container_name=container)
return self._get_chunk_id_and_fullpath(
chunk_inode, chunk_pos, content, chunk_id=chunk_id)
except Exception as exc:
self.logger.warn(
'chunk_id=%s chunk_pos=%s object=%s/%s/%s/%s/%s/%s: %s',
chunk_id, chunk_pos, str(account), str(container),
container_id, path, str(version), str(content_id), exc)
# version must be integer
try:
version = str(int(version))
except Exception:
version = None
try:
content = self.content_factory.get_by_path_and_version(
container_id, path, version,
account=account, container_name=container)
except ContentNotFound:
raise OrphanChunk('Content not found: possible orphan chunk')
return self._get_chunk_id_and_fullpath(
chunk_inode, chunk_pos, content, chunk_id=chunk_id)
def move_chunk(self, chunk_id):
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None:
raise OrphanChunk("Chunk not found in content")
other_chunks = self.chunks.filter(
metapos=current_chunk.metapos).exclude(id=chunk_id).all()
spare_urls = self._get_spare_chunk(other_chunks, [current_chunk])
self.logger.debug("copy chunk from %s to %s",
current_chunk.url, spare_urls[0])
self.blob_client.chunk_copy(
current_chunk.url, spare_urls[0], chunk_id=chunk_id,
fullpath=self.full_path, cid=self.container_id,
path=self.path, version=self.version, content_id=self.content_id)
self._update_spare_chunk(current_chunk, spare_urls[0])
try:
self.blob_client.chunk_delete(current_chunk.url)
except Exception as err:
self.logger.warn(
"Failed to delete chunk %s: %s", current_chunk.url, err)
current_chunk.url = spare_urls[0]
return current_chunk.raw()
def chunk_rebuild(self, container_id, content_id, chunk_id_or_pos):
self.logger.info('Rebuilding (container %s, content %s, chunk %s)',
container_id, content_id, chunk_id_or_pos)
try:
content = self.content_factory.get(container_id, content_id)
except ContentNotFound:
raise OrphanChunk('Content not found: possible orphan chunk')
chunk_size = 0
chunk_pos = None
if len(chunk_id_or_pos) < 32:
chunk_pos = chunk_id_or_pos
chunk_id = None
metapos = int(chunk_pos.split('.', 1)[0])
chunk_size = content.chunks.filter(metapos=metapos).all()[0].size
else:
if '/' in chunk_id_or_pos:
chunk_id = chunk_id_or_pos.rsplit('/', 1)[-1]
else:
chunk_id = chunk_id_or_pos
chunk = content.chunks.filter(id=chunk_id).one()
if chunk is None:
raise OrphanChunk(("Chunk not found in content:"
"possible orphan chunk"))
elif self.volume and chunk.host != self.volume:
raise ValueError("Chunk does not belong to this volume")
chunk_size = chunk.size
content.rebuild_chunk(chunk_id,
allow_same_rawx=self.allow_same_rawx,
chunk_pos=chunk_pos)
if self.try_chunk_delete:
try:
content.blob_client.chunk_delete(chunk.url)
self.logger.info("Chunk %s deleted", chunk.url)
except NotFound as exc:
self.logger.debug("Chunk %s: %s", chunk.url, exc)
# This call does not raise exception if chunk is not referenced
if chunk_id is not None:
self.rdir_client.chunk_delete(chunk.host, container_id, content_id,
chunk_id)
self.bytes_processed += chunk_size
self.total_bytes_processed += chunk_size
def rebuild_chunk(self, chunk_id):
# FIXME rebuild only the broken subchunk and not all broken
# subchunks in the metachunk.The current rainx rebuilds all
# subchunks. We can't download only the faulty chunk from the rainx
# without specifying all faulty chunks. Rainx sends only the data of
# the metachunk and not the parity data so we must rebuild metachunk
# through rainx services.
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None:
raise OrphanChunk("Chunk not found in content")
self.rebuild_metachunk(current_chunk.metapos,
force_broken_chunk=current_chunk)
def recover_chunk_fullpath(self, path, chunk_id=None):
if not chunk_id:
chunk_id = path.rsplit('/', 1)[-1]
# 1. Fetch chunk list from rdir (could be cached).
# Unfortunately we cannot seek for a chunk ID.
entries = [
x for x in self.rdir.chunk_fetch(self.volume_id, limit=-1)
if x[2] == chunk_id
]
if not entries:
raise KeyError('Chunk %s not found in rdir' % chunk_id)
elif len(entries) > 1:
self.logger.info('Chunk %s appears in %d objects', chunk_id,
len(entries))
# 2. Find content and container IDs
cid, content_id = entries[0][0:2]
# 3a. Call ContainerClient.content_locate()
# with the container ID and content ID
try:
meta, chunks = self.container_client.content_locate(
cid=cid, content=content_id)
except NotFound as err:
raise OrphanChunk('Cannot check %s is valid: %s' % (path, err))
# 3b. Resolve container ID into account and container names.
# FIXME(FVE): get account and container names from meta1
cmeta = self.container_client.container_get_properties(cid=cid)
aname = cmeta['system']['sys.account']
cname = cmeta['system']['sys.user.name']
fullpath = encode_fullpath(aname, cname, meta['name'], meta['version'],
content_id)
# 4. Check if the chunk actually belongs to the object
chunk_url = 'http://%s/%s' % (self.volume_id, chunk_id)
if chunk_url not in [x['url'] for x in chunks]:
raise OrphanChunk('Chunk %s not found in object %s' %
(chunk_url, fullpath))
# 5. Regenerate the fullpath
with open(path, 'w') as fd:
set_fullpath_xattr(fd, {chunk_id: fullpath})
return True
def move_linked_chunk(self, chunk_id, from_url):
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None:
raise OrphanChunk("Chunk not found in content")
_, to_url = self.blob_client.chunk_link(from_url, None, self.full_path)
self.logger.debug("link chunk %s from %s to %s", chunk_id, from_url,
to_url)
self._update_spare_chunk(current_chunk, to_url)
try:
self.blob_client.chunk_delete(current_chunk.url)
except Exception as err:
self.logger.warn(
"Failed to delete chunk %s: %s", current_chunk.url, err)
current_chunk.url = to_url
return current_chunk.raw()
def chunk_rebuild(self, container_id, content_id, chunk_id):
self.logger.info('Rebuilding (container %s, content %s, chunk %s)',
container_id, content_id, chunk_id)
try:
content = self.content_factory.get(container_id, content_id)
except ContentNotFound:
raise exc.OrphanChunk('Content not found')
chunk = content.chunks.filter(id=chunk_id).one()
if chunk is None:
raise OrphanChunk("Chunk not found in content")
chunk_size = chunk.size
content.rebuild_chunk(chunk_id, allow_same_rawx=self.allow_same_rawx)
self.rdir_client.chunk_delete(self.volume, container_id, content_id,
chunk_id)
self.bytes_processed += chunk_size
self.total_bytes_processed += chunk_size
def move_chunk(self, chunk_id):
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None:
raise OrphanChunk("Chunk not found in content")
other_chunks = self.chunks.filter(
metapos=current_chunk.metapos).exclude(id=chunk_id).all()
spare_urls = self._get_spare_chunk(other_chunks, [current_chunk])
self.logger.debug("copy chunk from %s to %s", current_chunk.url,
spare_urls[0])
self.blob_client.chunk_copy(current_chunk.url, spare_urls[0])
self._update_spare_chunk(current_chunk, spare_urls[0])
try:
self.blob_client.chunk_delete(current_chunk.url)
except:
self.logger.warn("Failed to delete chunk %s" % current_chunk.url)
current_chunk.url = spare_urls[0]
return current_chunk.raw()
def rebuild_chunk(self,
chunk_id,
allow_same_rawx=False,
chunk_pos=None,
allow_frozen_container=False):
# Identify the chunk to rebuild
current_chunk = self.chunks.filter(id=chunk_id).one()
if current_chunk is None and chunk_pos is None:
raise OrphanChunk("Chunk not found in content")
elif current_chunk is None:
current_chunk = self.chunks.filter(pos=chunk_pos).one()
if current_chunk is None:
chunk = {'pos': chunk_pos, 'url': ''}
current_chunk = Chunk(chunk)
else:
chunk_id = current_chunk.id
self.logger.debug('Chunk at pos %s has id %s', chunk_pos,
chunk_id)
chunks = self.chunks.filter(metapos=current_chunk.metapos)\
.exclude(id=chunk_id, pos=chunk_pos)
if chunk_id is None:
current_chunk.size = chunks[0].size
current_chunk.checksum = chunks[0].checksum
# Find a spare chunk address
broken_list = list()
if not allow_same_rawx and chunk_id is not None:
broken_list.append(current_chunk)
spare_url, _quals = self._get_spare_chunk(chunks.all(), broken_list)
new_chunk = Chunk({'pos': current_chunk.pos, 'url': spare_url[0]})
# Regenerate the lost chunk's data, from existing chunks
handler = ECRebuildHandler(chunks.raw(), current_chunk.subpos,
self.storage_method)
stream = handler.rebuild()
# Actually create the spare chunk
meta = {}
meta['chunk_id'] = new_chunk.id
meta['chunk_pos'] = current_chunk.pos
meta['container_id'] = self.container_id
# FIXME: should be 'content_chunkmethod' everywhere
# but sadly it isn't
meta['chunk_method'] = self.chunk_method
# FIXME: should be 'content_id' everywhere
# but sadly it isn't
meta['id'] = self.content_id
meta['content_path'] = self.path
# FIXME: should be 'content_policy' everywhere
# but sadly it isn't
meta['policy'] = self.policy
# FIXME: should be 'content_version' everywhere
# but sadly it isn't
meta['version'] = self.version
meta['metachunk_hash'] = current_chunk.checksum
meta['metachunk_size'] = current_chunk.size
meta['full_path'] = self.full_path
meta['oio_version'] = OIO_VERSION
self.blob_client.chunk_put(spare_url[0], meta, GeneratorIO(stream))
# Register the spare chunk in object's metadata
if chunk_id is None:
self._add_raw_chunk(current_chunk,
spare_url[0],
frozen=allow_frozen_container)
else:
self._update_spare_chunk(current_chunk,
spare_url[0],
frozen=allow_frozen_container)
self.logger.debug('Chunk %s repaired in %s', chunk_id or chunk_pos,
spare_url[0])
def rebuild(self,
container_id,
content_id,
chunk_id_or_pos,
rawx_id=None,
try_chunk_delete=False,
allow_frozen_container=True,
allow_same_rawx=True):
"""
Try to find the chunk in the metadata of the specified object,
then rebuild it.
"""
try:
content = self.content_factory.get(container_id, content_id)
except ContentNotFound:
raise OrphanChunk('Content not found: possible orphan chunk')
chunk_pos = None
if looks_like_chunk_position(chunk_id_or_pos):
chunk_pos = chunk_id_or_pos
chunk_id = None
else:
if '/' in chunk_id_or_pos:
parsed = urlparse(chunk_id_or_pos)
chunk_id = parsed.path.lstrip('/')
rawx_id = parsed.netloc
else:
chunk_id = chunk_id_or_pos
candidates = content.chunks.filter(id=chunk_id)
# FIXME(FVE): if for some reason the chunks have been registered
# with an IP address and port instead of an ID, this won't work.
if rawx_id:
candidates = candidates.filter(host=rawx_id)
chunk = candidates.one()
if chunk is None:
raise OrphanChunk(
'Chunk not found in content: possible orphan chunk: ' +
'%s' % (candidates.all(), ))
elif rawx_id and chunk.host != rawx_id:
raise ValueError('Chunk does not belong to this rawx')
rebuilt_bytes = content.rebuild_chunk(
chunk_id,
service_id=rawx_id,
allow_frozen_container=allow_frozen_container,
allow_same_rawx=allow_same_rawx,
chunk_pos=chunk_pos)
if try_chunk_delete:
try:
content.blob_client.chunk_delete(chunk.url)
self.logger.info("Old chunk %s deleted", chunk.url)
except Exception as exc:
self.logger.warn('Failed to delete old chunk %s: %s',
chunk.url, exc)
# This call does not raise exception if chunk is not referenced
if chunk_id is not None:
try:
self.rdir_client.chunk_delete(chunk.host, container_id,
content_id, chunk_id)
except Exception as exc:
self.logger.warn(
'Failed to delete chunk entry (%s) from the rdir (%s): %s',
chunk_id, chunk.host, exc)
return rebuilt_bytes
def rebuild_chunk(self,
chunk_id,
service_id=None,
allow_same_rawx=False,
chunk_pos=None,
allow_frozen_container=False):
# Identify the chunk to rebuild
candidates = self.chunks.filter(id=chunk_id)
if service_id is not None:
candidates = candidates.filter(host=service_id)
current_chunk = candidates.one()
if current_chunk is None and chunk_pos is None:
raise OrphanChunk("Chunk not found in content")
if current_chunk is None:
current_chunk = self.chunks.filter(pos=chunk_pos).one()
if current_chunk is None:
chunk = {'pos': chunk_pos, 'url': ''}
current_chunk = Chunk(chunk)
else:
chunk_id = current_chunk.id
self.logger.debug('Chunk at pos %s has id %s', chunk_pos,
chunk_id)
# Sort chunks by score to try to rebuild with higher score.
# When scores are close together (e.g. [95, 94, 94, 93, 50]),
# don't always start with the highest element.
chunks = self.chunks \
.filter(metapos=current_chunk.metapos) \
.exclude(id=chunk_id, pos=chunk_pos) \
.sort(key=lambda chunk: _get_weighted_random_score(chunk.raw()),
reverse=True)
if chunk_id is None:
current_chunk.size = chunks[0].size
current_chunk.checksum = chunks[0].checksum
# Find a spare chunk address
broken_list = list()
if not allow_same_rawx and chunk_id is not None:
broken_list.append(current_chunk)
spare_url, _quals = self._get_spare_chunk(chunks.all(),
broken_list,
position=current_chunk.pos)
new_chunk = Chunk({'pos': current_chunk.pos, 'url': spare_url[0]})
# Regenerate the lost chunk's data, from existing chunks
handler = ECRebuildHandler(chunks.raw(), current_chunk.subpos,
self.storage_method)
expected_chunk_size, stream = handler.rebuild()
# Actually create the spare chunk
meta = {}
meta['chunk_id'] = new_chunk.id
meta['chunk_pos'] = current_chunk.pos
meta['container_id'] = self.container_id
# FIXME: should be 'content_chunkmethod' everywhere
# but sadly it isn't
meta['chunk_method'] = self.chunk_method
# FIXME: should be 'content_id' everywhere
# but sadly it isn't
meta['id'] = self.content_id
meta['content_path'] = self.path
# FIXME: should be 'content_policy' everywhere
# but sadly it isn't
meta['policy'] = self.policy
# FIXME: should be 'content_version' everywhere
# but sadly it isn't
meta['version'] = self.version
meta['metachunk_hash'] = current_chunk.checksum
meta['metachunk_size'] = current_chunk.size
meta['full_path'] = self.full_path
meta['oio_version'] = OIO_VERSION
bytes_transferred, _ = self.blob_client.chunk_put(
spare_url[0], meta, GeneratorIO(stream, sub_generator=PY2))
if expected_chunk_size is not None \
and bytes_transferred != expected_chunk_size:
try:
self.blob_client.chunk_delete(spare_url[0])
except Exception as exc:
self.logger.warning(
'Failed to rollback the rebuild of the chunk: %s', exc)
raise ChunkException('The rebuilt chunk is not the correct size')
# Register the spare chunk in object's metadata
if chunk_id is None:
self._add_raw_chunk(current_chunk,
spare_url[0],
frozen=allow_frozen_container)
else:
self._update_spare_chunk(current_chunk,
spare_url[0],
frozen=allow_frozen_container)
self.logger.debug('Chunk %s repaired in %s', chunk_id or chunk_pos,
spare_url[0])
return bytes_transferred
