def recover_chunk_fullpath(self, path, chunk_id=None): if not chunk_id: chunk_id = path.rsplit('/', 1)[-1] # 1. Fetch chunk list from rdir (could be cached). # Unfortunately we cannot seek for a chunk ID. entries = [ x for x in self.rdir.chunk_fetch(self.volume_id, limit=-1) if x[2] == chunk_id ] if not entries: raise KeyError('Chunk %s not found in rdir' % chunk_id) elif len(entries) > 1: self.logger.info('Chunk %s appears in %d objects', chunk_id, len(entries)) # 2. Find content and container IDs cid, content_id = entries[0][0:2] # 3a. Call ContainerClient.content_locate() # with the container ID and content ID try: meta, chunks = self.container_client.content_locate( cid=cid, content=content_id) except NotFound as err: raise OrphanChunk('Cannot check %s is valid: %s' % (path, err)) # 3b. Resolve container ID into account and container names. # FIXME(FVE): get account and container names from meta1 cmeta = self.container_client.container_get_properties(cid=cid) aname = cmeta['system']['sys.account'] cname = cmeta['system']['sys.user.name'] fullpath = encode_fullpath(aname, cname, meta['name'], meta['version'], content_id) # 4. Check if the chunk actually belongs to the object chunk_url = 'http://%s/%s' % (self.volume_id, chunk_id) if chunk_url not in [x['url'] for x in chunks]: raise OrphanChunk('Chunk %s not found in object %s' % (chunk_url, fullpath)) # 5. Regenerate the fullpath with open(path, 'w') as fd: set_fullpath_xattr(fd, {chunk_id: fullpath}) return True
def convert_chunk(self, fd, chunk_id): meta, raw_meta = read_chunk_metadata(fd, chunk_id, for_conversion=True) links = meta.get('links', dict()) for chunk_id2, fullpath2 in links.items(): self.decode_fullpath(fullpath2) fullpath = meta.get('full_path') if fullpath is not None: self.decode_fullpath(fullpath) if meta.get('oio_version') == OIO_VERSION: return True, meta raw_chunk_id = None chunk_id = chunk_id.upper() chunk_pos = meta['chunk_pos'] container_id = meta['container_id'].upper() path = meta['content_path'] version = meta['content_version'] content_id = meta['content_id'].upper() new_fullpaths = dict() xattr_to_remove = list() success = True for k, v in raw_meta.items(): # fetch raw chunk ID if k == XATTR_CHUNK_ID: raw_chunk_id = v.upper() # search old fullpaths if not k.startswith(XATTR_OLD_FULLPATH) \ or not is_hexa(k[4:], size=64): continue try: account2, container2, container_id2, path2, version2, \ content_id2 = self.decode_old_fullpath(v) if meta['chunk_id'] == chunk_id \ and container_id == container_id2 \ and path == path2 \ and version == version2: if content_id2 is None: content_id2 = self.content_id_from_name(container_id2, path2, version2, search=True) chunk_id2, new_fullpath = self.encode_fullpath( fd, chunk_id, account2, container2, path2, version2, content_id2) new_fullpaths[chunk_id2] = new_fullpath else: chunk_id2, new_fullpath = self.get_chunk_id_and_fullpath( fd, chunk_pos, container_id2, path2, version2, account=account2, container=container2, content_id=content_id2) new_fullpaths[chunk_id2] = new_fullpath xattr_to_remove.append(k) except Exception as exc: success = False self.logger.warn('chunk_id=%s old_fullpath=%s: %s', chunk_id, k, exc) # old xattr if raw_chunk_id is not None: try: if raw_chunk_id != chunk_id and raw_chunk_id not in links: if raw_chunk_id not in new_fullpaths: meta2, _ = read_chunk_metadata(fd, raw_chunk_id) container_id2 = meta2['container_id'].upper() path2 = meta2['content_path'] version2 = meta2['content_version'] content_id2 = meta2['content_id'].upper() raw_chunk_id2, new_fullpath = \ self.get_chunk_id_and_fullpath( fd, chunk_pos, container_id2, path2, version2, chunk_id=raw_chunk_id, content_id=content_id2) new_fullpaths[raw_chunk_id2] = new_fullpath elif raw_chunk_id == chunk_id and fullpath is None: if raw_chunk_id not in new_fullpaths: raw_chunk_id2, new_fullpath = \ self.get_chunk_id_and_fullpath( fd, chunk_pos, container_id, path, version, chunk_id=raw_chunk_id, content_id=content_id) new_fullpaths[raw_chunk_id2] = new_fullpath except Exception as exc: success = False self.logger.warn('chunk_id=%s (old xattr): %s', raw_chunk_id, exc) self.save_xattr(fd, chunk_id, raw_meta) if self.dry_run: self.logger.info( "[dryrun] Converting chunk %s: success=%s new_fullpaths=%s " "xattr_to_remove=%s", chunk_id, str(success), str(new_fullpaths), str(xattr_to_remove)) else: # for security, if there is an error, we don't delete old xattr set_fullpath_xattr(fd, new_fullpaths, success, xattr_to_remove) return success, None