def read_chunk_metadata(fd, chunk_id, check_chunk_id=True): chunk_id = chunk_id.upper() raw_meta = read_user_xattr(fd) raw_meta_copy = None meta = {} meta['links'] = dict() raw_chunk_id = container_id = path = version = content_id = None for k, v in raw_meta.iteritems(): # New chunk if k.startswith(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX): chunkid = k[len(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX):] if chunkid == chunk_id: raw_chunk_id = chunkid meta['full_path'] = v account, container, path, version, content_id = \ decode_fullpath(v) container_id = cid_from_name(account, container) else: meta['links'][chunkid] = v if raw_chunk_id: raw_meta_copy = raw_meta.copy() raw_meta[chunk_xattr_keys['chunk_id']] = raw_chunk_id raw_meta[chunk_xattr_keys['container_id']] = container_id raw_meta[chunk_xattr_keys['content_path']] = path raw_meta[chunk_xattr_keys['content_version']] = version raw_meta[chunk_xattr_keys['content_id']] = content_id for k, v in chunk_xattr_keys.iteritems(): if v not in raw_meta: if k not in chunk_xattr_keys_optional: raise exc.MissingAttribute(v) else: meta[k] = raw_meta[v] if check_chunk_id and meta['chunk_id'] != chunk_id: raise exc.MissingAttribute(chunk_xattr_keys['chunk_id']) return meta, raw_meta_copy if raw_meta_copy else raw_meta
def decode_fullpath(self, fullpath): account, container, path, version, content_id = decode_fullpath( fullpath) cid = self.cid_from_name(account, container) cid, path, version, content_id = self._save_content( cid, path, version, content_id) return account, container, cid, path, version, content_id
def chunk_move(self, path, chunk_id): meta = self.load_chunk_metadata(path, chunk_id) container_id = meta['container_id'] content_id = meta['content_id'] chunk_id = meta['chunk_id'] try: content = self.content_factory.get(container_id, content_id) except ContentNotFound: raise exc.OrphanChunk('Content not found') new_chunk = content.move_chunk(chunk_id) self.logger.info('moved chunk http://%s/%s to %s', self.address, chunk_id, new_chunk['url']) if self.allow_links: old_links = meta['links'] for chunk_id, fullpath in old_links.iteritems(): account, container, _, _, content_id = \ decode_fullpath(fullpath) container_id = cid_from_name(account, container) try: content = self.content_factory.get(container_id, content_id) except ContentNotFound: raise exc.OrphanChunk('Content not found') new_linked_chunk = content.move_linked_chunk( chunk_id, new_chunk['url']) self.logger.info('moved chunk http://%s/%s to %s', self.address, chunk_id, new_linked_chunk['url'])
def link(self): """ Create new hard links for all the chunks of a metachunk. """ new_meta_chunks = list() failed_chunks = list() # pylint: disable=unbalanced-tuple-unpacking acct, ct, path, vers, _ = decode_fullpath(self.fullpath) cid = cid_from_name(acct, ct) for chunk_target in self.meta_chunk_target: try: chunk_id = compute_chunk_id(cid, path, vers, chunk_target['pos'], self.policy) resp, new_chunk_url = self.blob_client.chunk_link( chunk_target['url'], chunk_id, self.fullpath, connection_timeout=self.connection_timeout, write_timeout=self.write_timeout, reqid=self.reqid, perfdata=self.perfdata, logger=self.logger) new_chunk = chunk_target.copy() new_chunk['url'] = new_chunk_url new_meta_chunks.append(new_chunk) except Exception: failed_chunks.append(chunk_target) try: self.quorum_or_fail(new_meta_chunks, failed_chunks) except Exception as ex: raise exc.UnfinishedUploadException(ex, new_meta_chunks) return new_meta_chunks
def decode_fullpath(self, fullpath): # pylint: disable=unbalanced-tuple-unpacking account, container, path, version, content_id = decode_fullpath( fullpath) cid = self.cid_from_name(account, container) cid, path, version, content_id = self._save_content( cid, path, version, content_id) return account, container, cid, path, version, content_id
def decode_old_fullpath(self, old_fullpath): # pylint: disable=unbalanced-tuple-unpacking try: account, container, path, version = decode_old_fullpath( old_fullpath) cid = self.cid_from_name(account, container) content_id = self.content_id_from_name(cid, path, version) except ValueError: # We never know, let's try to decode the fullpath as if it was new account, container, path, version, content_id = decode_fullpath( old_fullpath) cid = self.cid_from_name(account, container) return account, container, cid, path, version, content_id
def chunk_move(self, path, chunk_id): meta = self.load_chunk_metadata(path, chunk_id) container_id = meta['container_id'] content_id = meta['content_id'] chunk_id = meta['chunk_id'] # Maybe skip the chunk because it doesn't match the size constaint chunk_size = int(meta['chunk_size']) min_chunk_size = int(self.conf.get('min_chunk_size', 0)) max_chunk_size = int(self.conf.get('max_chunk_size', 0)) if chunk_size < min_chunk_size: self.logger.debug("SKIP %s too small", path) return if max_chunk_size > 0 and chunk_size > max_chunk_size: self.logger.debug("SKIP %s too big", path) return # Start moving the chunk try: content = self.content_factory.get(container_id, content_id) except ContentNotFound: raise exc.OrphanChunk('Content not found') new_chunk = content.move_chunk( chunk_id, service_id=self.service_id, fake_excluded_chunks=self.fake_excluded_chunks) self.logger.info('moved chunk http://%s/%s to %s', self.service_id, chunk_id, new_chunk['url']) if self.allow_links: old_links = meta['links'] for chunk_id, fullpath in old_links.items(): # pylint: disable=unbalanced-tuple-unpacking account, container, _, _, content_id = \ decode_fullpath(fullpath) container_id = cid_from_name(account, container) try: content = self.content_factory.get(container_id, content_id) except ContentNotFound: raise exc.OrphanChunk('Content not found') new_linked_chunk = content.move_linked_chunk( chunk_id, new_chunk['url']) self.logger.info('moved chunk http://%s/%s to %s', self.service_id, chunk_id, new_linked_chunk['url'])
def read_chunk_metadata(fd, chunk_id, check_chunk_id=True): chunk_id = chunk_id.upper() raw_meta = read_user_xattr(fd) raw_meta_copy = None meta = {} meta['links'] = dict() attr_vers = 0.0 raw_chunk_id = container_id = path = version = content_id = None missing = list() for k, v in raw_meta.iteritems(): # New chunks have a version if k == chunk_xattr_keys['oio_version']: attr_vers = float(v) # Chunks with version >= 4.2 have a "full_path" elif k.startswith(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX): parsed_chunk_id = k[len(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX):] if parsed_chunk_id == chunk_id: raw_chunk_id = parsed_chunk_id meta['full_path'] = v account, container, path, version, content_id = \ decode_fullpath(v) container_id = cid_from_name(account, container) else: meta['links'][parsed_chunk_id] = v if raw_chunk_id: raw_meta_copy = raw_meta.copy() raw_meta[chunk_xattr_keys['chunk_id']] = raw_chunk_id raw_meta[chunk_xattr_keys['container_id']] = container_id raw_meta[chunk_xattr_keys['content_path']] = path raw_meta[chunk_xattr_keys['content_version']] = version raw_meta[chunk_xattr_keys['content_id']] = content_id if attr_vers >= 4.2 and 'full_path' not in meta: # TODO(FVE): in that case, do not warn about other attributes # that could be deduced from this one. missing.append( exc.MissingAttribute(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + chunk_id)) for k, v in chunk_xattr_keys.iteritems(): if v not in raw_meta: if k not in chunk_xattr_keys_optional: missing.append(exc.MissingAttribute(v)) else: meta[k] = raw_meta[v] if missing: raise exc.FaultyChunk(*missing) if check_chunk_id and meta['chunk_id'] != chunk_id: raise exc.MissingAttribute(chunk_xattr_keys['chunk_id']) return meta, raw_meta_copy if raw_meta_copy else raw_meta
def complete_target_from_chunk_metadata(self, target, xattr_meta): """ Complete a Target object from metadata found in chunk's extended attributes. In case the "fullpath" is not available, try to read legacy metadata, and maybe ask meta1 to resolve the CID into account and container names. """ # pylint: disable=unbalanced-tuple-unpacking try: acct, ct, path, vers, content_id = \ decode_fullpath(xattr_meta['full_path']) target.account = acct target.container = ct target.obj = path target.content_id = content_id target.version = vers except KeyError: # No fullpath header, try legacy headers if 'content_path' in xattr_meta: target.obj = xattr_meta['content_path'] if 'content_id' in xattr_meta: target.content_id = xattr_meta['content_id'] if 'content_version' in xattr_meta: target.version = xattr_meta['content_version'] cid = xattr_meta.get('container_id') if cid: try: md = self.api.directory.show(cid=cid) acct = md.get('account') ct = md.get('name') if acct: target.account = acct if ct: target.container = ct except Exception as err: self.logger.warn( "Failed to resolve CID %s into account " "and container names: %s", cid, err)
def test_decode_with_utf8_info(self): info = decode_fullpath( "m%C5%B7account/mycontain%C3%A9r/my%C3%B6bject/9876543210/" "0123456789ABCDEF") self.assertEqual(info, ("mŷaccount", "mycontainér", "myöbject", "9876543210", "0123456789ABCDEF"))
def test_decode(self): info = decode_fullpath( "myaccount/mycontainer/myobject/9876543210/0123456789ABCDEF") self.assertEqual(info, ("myaccount", "mycontainer", "myobject", "9876543210", "0123456789ABCDEF"))