def rebuild(self, container_id, content_id, chunk_id_or_pos, rawx_id=None, try_chunk_delete=False, allow_same_rawx=True): """ Try to find the chunk in the metadata of the specified object, then rebuild it. """ try: content = self.content_factory.get(container_id, content_id) except ContentNotFound: raise OrphanChunk('Content not found: possible orphan chunk') chunk_size = 0 chunk_pos = None if len(chunk_id_or_pos) < 32: chunk_pos = chunk_id_or_pos chunk_id = None metapos = int(chunk_pos.split('.', 1)[0]) chunk_size = content.chunks.filter(metapos=metapos).all()[0].size else: if '/' in chunk_id_or_pos: chunk_id = chunk_id_or_pos.rsplit('/', 1)[-1] else: chunk_id = chunk_id_or_pos chunk = content.chunks.filter(id=chunk_id).one() if chunk is None: raise OrphanChunk( 'Chunk not found in content: possible orphan chunk') elif rawx_id and chunk.host != rawx_id: raise ValueError('Chunk does not belong to this rawx') chunk_size = chunk.size content.rebuild_chunk(chunk_id, allow_same_rawx=allow_same_rawx, chunk_pos=chunk_pos) if try_chunk_delete: try: content.blob_client.chunk_delete(chunk.url) self.logger.info("Old chunk %s deleted", chunk.url) except Exception as exc: self.logger.warn('Failed to delete old chunk %s: %s', chunk.url, exc) # This call does not raise exception if chunk is not referenced if chunk_id is not None: try: self.rdir_client.chunk_delete(chunk.host, container_id, content_id, chunk_id) except Exception as exc: self.logger.warn( 'Failed to delete chunk entry (%s) from the rdir (%s): %s', chunk_id, chunk.host, exc) return chunk_size
def encode_fullpath(self, chunk_inode, chunk_id, account, container, path, version, content_id): # check if chunk exists and has the same inode if not is_hexa(chunk_id) or len(chunk_id) != STRLEN_CHUNKID: raise ValueError('chunk ID must be hexadecimal (%s)' % STRLEN_CHUNKID) try: chunk_inode2 = os.stat(self._get_path(chunk_id)).st_ino except OSError: raise OrphanChunk('No such chunk: possible orphan chunk') if chunk_inode2 != chunk_inode: raise OrphanChunk('Not the same inode: possible orphan chunk') # check fullpath and chunk ID if isinstance(version, basestring): try: version = int(version) except ValueError: raise ValueError('version must be a number') if version <= 0: raise ValueError('version must be positive') if not is_hexa(content_id): raise ValueError('content ID must be hexadecimal') fullpath = encode_fullpath(account, container, path, version, content_id.upper()) return chunk_id.upper(), fullpath
def rebuild_chunk(self, chunk_id, allow_same_rawx=False, chunk_pos=None): current_chunk = self.chunks.filter(id=chunk_id).one() if current_chunk is None and chunk_pos is None: raise OrphanChunk("Chunk not found in content") elif current_chunk is None: chunk = {"pos": chunk_pos, "url": ""} current_chunk = Chunk(chunk) chunks = self.chunks.filter(metapos=current_chunk.metapos)\ .exclude(id=chunk_id) if chunk_id is None: current_chunk.size = chunks[0].size current_chunk.checksum = chunks[0].checksum broken_list = list() if not allow_same_rawx: broken_list.append(current_chunk) spare_url = self._get_spare_chunk(chunks.all(), broken_list) handler = ECRebuildHandler(chunks.raw(), current_chunk.subpos, self.storage_method) new_chunk = {'pos': current_chunk.pos, 'url': spare_url[0]} new_chunk = Chunk(new_chunk) stream = handler.rebuild() meta = {} meta['chunk_id'] = new_chunk.id meta['chunk_pos'] = current_chunk.pos meta['container_id'] = self.container_id # FIXME: should be 'content_chunkmethod' everywhere # but sadly it isn't meta['chunk_method'] = self.chunk_method # FIXME: should be 'content_id' everywhere # but sadly it isn't meta['id'] = self.content_id meta['content_path'] = self.path # FIXME: should be 'content_policy' everywhere # but sadly it isn't meta['policy'] = self.policy # FIXME: should be 'content_version' everywhere # but sadly it isn't meta['version'] = self.version meta['metachunk_hash'] = current_chunk.checksum meta['metachunk_size'] = current_chunk.size meta['full_path'] = self.full_path meta['oio_version'] = OIO_VERSION self.blob_client.chunk_put(spare_url[0], meta, GeneratorIO(stream)) if chunk_id is None: self._add_raw_chunk(current_chunk, spare_url[0]) else: self._update_spare_chunk(current_chunk, spare_url[0])
def _get_chunk_id_and_fullpath(self, chunk_inode, chunk_pos, content, chunk_id=None): content.container_id, content.account, content.container_name = \ self._save_container(content.container_id, content.account, content.container_name) content.container_id, content.path, content.version, \ content.content_id = self._save_content( content.container_id, content.path, content.version, content.content_id) chunks = content.chunks.filter(host=self.volume_id) if chunk_id: chunks = chunks.filter(id=chunk_id) chunk = chunks.filter(pos=chunk_pos).one() if chunk is None: raise OrphanChunk('Chunk not found in content:' 'possible orphan chunk') chunk_id, new_fullpath = self.encode_fullpath( chunk_inode, chunk.id, content.account, content.container_name, content.path, content.version, content.content_id) return chunk_id, new_fullpath
def rebuild_chunk(self, chunk_id): current_chunk = self.chunks.filter(id=chunk_id).one() if current_chunk is None: raise OrphanChunk("Chunk not found in content") chunks = self.chunks.filter(metapos=current_chunk.metapos)\ .exclude(id=chunk_id) spare_url = self._get_spare_chunk(chunks.all(), [current_chunk]) handler = ECRebuildHandler( chunks.raw(), current_chunk.subpos, self.storage_method) new_chunk = {'pos': current_chunk.pos, 'url': spare_url[0]} new_chunk = Chunk(new_chunk) stream = handler.rebuild() meta = {} meta['chunk_id'] = new_chunk.id meta['chunk_pos'] = current_chunk.pos meta['container_id'] = self.container_id meta['content_chunkmethod'] = self.chunk_method meta['content_id'] = self.content_id meta['content_path'] = self.path meta['content_policy'] = self.stgpol meta['content_version'] = self.version meta['metachunk_hash'] = current_chunk.checksum meta['metachunk_size'] = current_chunk.size self.blob_client.chunk_put(spare_url[0], meta, stream) self._update_spare_chunk(current_chunk, spare_url[0])
def get_chunk_id_and_fullpath( self, chunk_inode, chunk_pos, container_id, path, version, chunk_id=None, account=None, container=None, content_id=None): if account is None or container is None: account, container = self.name_from_cid(container_id) if content_id: try: content = self.content_factory.get( container_id, content_id, account=account, container_name=container) return self._get_chunk_id_and_fullpath( chunk_inode, chunk_pos, content, chunk_id=chunk_id) except Exception as exc: self.logger.warn( 'chunk_id=%s chunk_pos=%s object=%s/%s/%s/%s/%s/%s: %s', chunk_id, chunk_pos, str(account), str(container), container_id, path, str(version), str(content_id), exc) # version must be integer try: version = str(int(version)) except Exception: version = None try: content = self.content_factory.get_by_path_and_version( container_id, path, version, account=account, container_name=container) except ContentNotFound: raise OrphanChunk('Content not found: possible orphan chunk') return self._get_chunk_id_and_fullpath( chunk_inode, chunk_pos, content, chunk_id=chunk_id)
def move_chunk(self, chunk_id): current_chunk = self.chunks.filter(id=chunk_id).one() if current_chunk is None: raise OrphanChunk("Chunk not found in content") other_chunks = self.chunks.filter( metapos=current_chunk.metapos).exclude(id=chunk_id).all() spare_urls = self._get_spare_chunk(other_chunks, [current_chunk]) self.logger.debug("copy chunk from %s to %s", current_chunk.url, spare_urls[0]) self.blob_client.chunk_copy( current_chunk.url, spare_urls[0], chunk_id=chunk_id, fullpath=self.full_path, cid=self.container_id, path=self.path, version=self.version, content_id=self.content_id) self._update_spare_chunk(current_chunk, spare_urls[0]) try: self.blob_client.chunk_delete(current_chunk.url) except Exception as err: self.logger.warn( "Failed to delete chunk %s: %s", current_chunk.url, err) current_chunk.url = spare_urls[0] return current_chunk.raw()
def chunk_rebuild(self, container_id, content_id, chunk_id_or_pos): self.logger.info('Rebuilding (container %s, content %s, chunk %s)', container_id, content_id, chunk_id_or_pos) try: content = self.content_factory.get(container_id, content_id) except ContentNotFound: raise OrphanChunk('Content not found: possible orphan chunk') chunk_size = 0 chunk_pos = None if len(chunk_id_or_pos) < 32: chunk_pos = chunk_id_or_pos chunk_id = None metapos = int(chunk_pos.split('.', 1)[0]) chunk_size = content.chunks.filter(metapos=metapos).all()[0].size else: if '/' in chunk_id_or_pos: chunk_id = chunk_id_or_pos.rsplit('/', 1)[-1] else: chunk_id = chunk_id_or_pos chunk = content.chunks.filter(id=chunk_id).one() if chunk is None: raise OrphanChunk(("Chunk not found in content:" "possible orphan chunk")) elif self.volume and chunk.host != self.volume: raise ValueError("Chunk does not belong to this volume") chunk_size = chunk.size content.rebuild_chunk(chunk_id, allow_same_rawx=self.allow_same_rawx, chunk_pos=chunk_pos) if self.try_chunk_delete: try: content.blob_client.chunk_delete(chunk.url) self.logger.info("Chunk %s deleted", chunk.url) except NotFound as exc: self.logger.debug("Chunk %s: %s", chunk.url, exc) # This call does not raise exception if chunk is not referenced if chunk_id is not None: self.rdir_client.chunk_delete(chunk.host, container_id, content_id, chunk_id) self.bytes_processed += chunk_size self.total_bytes_processed += chunk_size
def rebuild_chunk(self, chunk_id): # FIXME rebuild only the broken subchunk and not all broken # subchunks in the metachunk.The current rainx rebuilds all # subchunks. We can't download only the faulty chunk from the rainx # without specifying all faulty chunks. Rainx sends only the data of # the metachunk and not the parity data so we must rebuild metachunk # through rainx services. current_chunk = self.chunks.filter(id=chunk_id).one() if current_chunk is None: raise OrphanChunk("Chunk not found in content") self.rebuild_metachunk(current_chunk.metapos, force_broken_chunk=current_chunk)
def recover_chunk_fullpath(self, path, chunk_id=None): if not chunk_id: chunk_id = path.rsplit('/', 1)[-1] # 1. Fetch chunk list from rdir (could be cached). # Unfortunately we cannot seek for a chunk ID. entries = [ x for x in self.rdir.chunk_fetch(self.volume_id, limit=-1) if x[2] == chunk_id ] if not entries: raise KeyError('Chunk %s not found in rdir' % chunk_id) elif len(entries) > 1: self.logger.info('Chunk %s appears in %d objects', chunk_id, len(entries)) # 2. Find content and container IDs cid, content_id = entries[0][0:2] # 3a. Call ContainerClient.content_locate() # with the container ID and content ID try: meta, chunks = self.container_client.content_locate( cid=cid, content=content_id) except NotFound as err: raise OrphanChunk('Cannot check %s is valid: %s' % (path, err)) # 3b. Resolve container ID into account and container names. # FIXME(FVE): get account and container names from meta1 cmeta = self.container_client.container_get_properties(cid=cid) aname = cmeta['system']['sys.account'] cname = cmeta['system']['sys.user.name'] fullpath = encode_fullpath(aname, cname, meta['name'], meta['version'], content_id) # 4. Check if the chunk actually belongs to the object chunk_url = 'http://%s/%s' % (self.volume_id, chunk_id) if chunk_url not in [x['url'] for x in chunks]: raise OrphanChunk('Chunk %s not found in object %s' % (chunk_url, fullpath)) # 5. Regenerate the fullpath with open(path, 'w') as fd: set_fullpath_xattr(fd, {chunk_id: fullpath}) return True
def move_linked_chunk(self, chunk_id, from_url): current_chunk = self.chunks.filter(id=chunk_id).one() if current_chunk is None: raise OrphanChunk("Chunk not found in content") _, to_url = self.blob_client.chunk_link(from_url, None, self.full_path) self.logger.debug("link chunk %s from %s to %s", chunk_id, from_url, to_url) self._update_spare_chunk(current_chunk, to_url) try: self.blob_client.chunk_delete(current_chunk.url) except Exception as err: self.logger.warn( "Failed to delete chunk %s: %s", current_chunk.url, err) current_chunk.url = to_url return current_chunk.raw()
def chunk_rebuild(self, container_id, content_id, chunk_id): self.logger.info('Rebuilding (container %s, content %s, chunk %s)', container_id, content_id, chunk_id) try: content = self.content_factory.get(container_id, content_id) except ContentNotFound: raise exc.OrphanChunk('Content not found') chunk = content.chunks.filter(id=chunk_id).one() if chunk is None: raise OrphanChunk("Chunk not found in content") chunk_size = chunk.size content.rebuild_chunk(chunk_id, allow_same_rawx=self.allow_same_rawx) self.rdir_client.chunk_delete(self.volume, container_id, content_id, chunk_id) self.bytes_processed += chunk_size self.total_bytes_processed += chunk_size
def move_chunk(self, chunk_id): current_chunk = self.chunks.filter(id=chunk_id).one() if current_chunk is None: raise OrphanChunk("Chunk not found in content") other_chunks = self.chunks.filter( metapos=current_chunk.metapos).exclude(id=chunk_id).all() spare_urls = self._get_spare_chunk(other_chunks, [current_chunk]) self.logger.debug("copy chunk from %s to %s", current_chunk.url, spare_urls[0]) self.blob_client.chunk_copy(current_chunk.url, spare_urls[0]) self._update_spare_chunk(current_chunk, spare_urls[0]) try: self.blob_client.chunk_delete(current_chunk.url) except: self.logger.warn("Failed to delete chunk %s" % current_chunk.url) current_chunk.url = spare_urls[0] return current_chunk.raw()
def rebuild_chunk(self, chunk_id, allow_same_rawx=False, chunk_pos=None, allow_frozen_container=False): # Identify the chunk to rebuild current_chunk = self.chunks.filter(id=chunk_id).one() if current_chunk is None and chunk_pos is None: raise OrphanChunk("Chunk not found in content") elif current_chunk is None: current_chunk = self.chunks.filter(pos=chunk_pos).one() if current_chunk is None: chunk = {'pos': chunk_pos, 'url': ''} current_chunk = Chunk(chunk) else: chunk_id = current_chunk.id self.logger.debug('Chunk at pos %s has id %s', chunk_pos, chunk_id) chunks = self.chunks.filter(metapos=current_chunk.metapos)\ .exclude(id=chunk_id, pos=chunk_pos) if chunk_id is None: current_chunk.size = chunks[0].size current_chunk.checksum = chunks[0].checksum # Find a spare chunk address broken_list = list() if not allow_same_rawx and chunk_id is not None: broken_list.append(current_chunk) spare_url, _quals = self._get_spare_chunk(chunks.all(), broken_list) new_chunk = Chunk({'pos': current_chunk.pos, 'url': spare_url[0]}) # Regenerate the lost chunk's data, from existing chunks handler = ECRebuildHandler(chunks.raw(), current_chunk.subpos, self.storage_method) stream = handler.rebuild() # Actually create the spare chunk meta = {} meta['chunk_id'] = new_chunk.id meta['chunk_pos'] = current_chunk.pos meta['container_id'] = self.container_id # FIXME: should be 'content_chunkmethod' everywhere # but sadly it isn't meta['chunk_method'] = self.chunk_method # FIXME: should be 'content_id' everywhere # but sadly it isn't meta['id'] = self.content_id meta['content_path'] = self.path # FIXME: should be 'content_policy' everywhere # but sadly it isn't meta['policy'] = self.policy # FIXME: should be 'content_version' everywhere # but sadly it isn't meta['version'] = self.version meta['metachunk_hash'] = current_chunk.checksum meta['metachunk_size'] = current_chunk.size meta['full_path'] = self.full_path meta['oio_version'] = OIO_VERSION self.blob_client.chunk_put(spare_url[0], meta, GeneratorIO(stream)) # Register the spare chunk in object's metadata if chunk_id is None: self._add_raw_chunk(current_chunk, spare_url[0], frozen=allow_frozen_container) else: self._update_spare_chunk(current_chunk, spare_url[0], frozen=allow_frozen_container) self.logger.debug('Chunk %s repaired in %s', chunk_id or chunk_pos, spare_url[0])
def rebuild(self, container_id, content_id, chunk_id_or_pos, rawx_id=None, try_chunk_delete=False, allow_frozen_container=True, allow_same_rawx=True): """ Try to find the chunk in the metadata of the specified object, then rebuild it. """ try: content = self.content_factory.get(container_id, content_id) except ContentNotFound: raise OrphanChunk('Content not found: possible orphan chunk') chunk_pos = None if looks_like_chunk_position(chunk_id_or_pos): chunk_pos = chunk_id_or_pos chunk_id = None else: if '/' in chunk_id_or_pos: parsed = urlparse(chunk_id_or_pos) chunk_id = parsed.path.lstrip('/') rawx_id = parsed.netloc else: chunk_id = chunk_id_or_pos candidates = content.chunks.filter(id=chunk_id) # FIXME(FVE): if for some reason the chunks have been registered # with an IP address and port instead of an ID, this won't work. if rawx_id: candidates = candidates.filter(host=rawx_id) chunk = candidates.one() if chunk is None: raise OrphanChunk( 'Chunk not found in content: possible orphan chunk: ' + '%s' % (candidates.all(), )) elif rawx_id and chunk.host != rawx_id: raise ValueError('Chunk does not belong to this rawx') rebuilt_bytes = content.rebuild_chunk( chunk_id, service_id=rawx_id, allow_frozen_container=allow_frozen_container, allow_same_rawx=allow_same_rawx, chunk_pos=chunk_pos) if try_chunk_delete: try: content.blob_client.chunk_delete(chunk.url) self.logger.info("Old chunk %s deleted", chunk.url) except Exception as exc: self.logger.warn('Failed to delete old chunk %s: %s', chunk.url, exc) # This call does not raise exception if chunk is not referenced if chunk_id is not None: try: self.rdir_client.chunk_delete(chunk.host, container_id, content_id, chunk_id) except Exception as exc: self.logger.warn( 'Failed to delete chunk entry (%s) from the rdir (%s): %s', chunk_id, chunk.host, exc) return rebuilt_bytes
def rebuild_chunk(self, chunk_id, service_id=None, allow_same_rawx=False, chunk_pos=None, allow_frozen_container=False): # Identify the chunk to rebuild candidates = self.chunks.filter(id=chunk_id) if service_id is not None: candidates = candidates.filter(host=service_id) current_chunk = candidates.one() if current_chunk is None and chunk_pos is None: raise OrphanChunk("Chunk not found in content") if current_chunk is None: current_chunk = self.chunks.filter(pos=chunk_pos).one() if current_chunk is None: chunk = {'pos': chunk_pos, 'url': ''} current_chunk = Chunk(chunk) else: chunk_id = current_chunk.id self.logger.debug('Chunk at pos %s has id %s', chunk_pos, chunk_id) # Sort chunks by score to try to rebuild with higher score. # When scores are close together (e.g. [95, 94, 94, 93, 50]), # don't always start with the highest element. chunks = self.chunks \ .filter(metapos=current_chunk.metapos) \ .exclude(id=chunk_id, pos=chunk_pos) \ .sort(key=lambda chunk: _get_weighted_random_score(chunk.raw()), reverse=True) if chunk_id is None: current_chunk.size = chunks[0].size current_chunk.checksum = chunks[0].checksum # Find a spare chunk address broken_list = list() if not allow_same_rawx and chunk_id is not None: broken_list.append(current_chunk) spare_url, _quals = self._get_spare_chunk(chunks.all(), broken_list, position=current_chunk.pos) new_chunk = Chunk({'pos': current_chunk.pos, 'url': spare_url[0]}) # Regenerate the lost chunk's data, from existing chunks handler = ECRebuildHandler(chunks.raw(), current_chunk.subpos, self.storage_method) expected_chunk_size, stream = handler.rebuild() # Actually create the spare chunk meta = {} meta['chunk_id'] = new_chunk.id meta['chunk_pos'] = current_chunk.pos meta['container_id'] = self.container_id # FIXME: should be 'content_chunkmethod' everywhere # but sadly it isn't meta['chunk_method'] = self.chunk_method # FIXME: should be 'content_id' everywhere # but sadly it isn't meta['id'] = self.content_id meta['content_path'] = self.path # FIXME: should be 'content_policy' everywhere # but sadly it isn't meta['policy'] = self.policy # FIXME: should be 'content_version' everywhere # but sadly it isn't meta['version'] = self.version meta['metachunk_hash'] = current_chunk.checksum meta['metachunk_size'] = current_chunk.size meta['full_path'] = self.full_path meta['oio_version'] = OIO_VERSION bytes_transferred, _ = self.blob_client.chunk_put( spare_url[0], meta, GeneratorIO(stream, sub_generator=PY2)) if expected_chunk_size is not None \ and bytes_transferred != expected_chunk_size: try: self.blob_client.chunk_delete(spare_url[0]) except Exception as exc: self.logger.warning( 'Failed to rollback the rebuild of the chunk: %s', exc) raise ChunkException('The rebuilt chunk is not the correct size') # Register the spare chunk in object's metadata if chunk_id is None: self._add_raw_chunk(current_chunk, spare_url[0], frozen=allow_frozen_container) else: self._update_spare_chunk(current_chunk, spare_url[0], frozen=allow_frozen_container) self.logger.debug('Chunk %s repaired in %s', chunk_id or chunk_pos, spare_url[0]) return bytes_transferred