def read_chunk_metadata(fd, chunk_id, check_chunk_id=True):
chunk_id = chunk_id.upper()
raw_meta = read_user_xattr(fd)
raw_meta_copy = None
meta = {}
meta['links'] = dict()
raw_chunk_id = container_id = path = version = content_id = None
for k, v in raw_meta.iteritems():
# New chunk
if k.startswith(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX):
chunkid = k[len(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX):]
if chunkid == chunk_id:
raw_chunk_id = chunkid
meta['full_path'] = v
account, container, path, version, content_id = \
decode_fullpath(v)
container_id = cid_from_name(account, container)
else:
meta['links'][chunkid] = v
if raw_chunk_id:
raw_meta_copy = raw_meta.copy()
raw_meta[chunk_xattr_keys['chunk_id']] = raw_chunk_id
raw_meta[chunk_xattr_keys['container_id']] = container_id
raw_meta[chunk_xattr_keys['content_path']] = path
raw_meta[chunk_xattr_keys['content_version']] = version
raw_meta[chunk_xattr_keys['content_id']] = content_id
for k, v in chunk_xattr_keys.iteritems():
if v not in raw_meta:
if k not in chunk_xattr_keys_optional:
raise exc.MissingAttribute(v)
else:
meta[k] = raw_meta[v]
if check_chunk_id and meta['chunk_id'] != chunk_id:
raise exc.MissingAttribute(chunk_xattr_keys['chunk_id'])
return meta, raw_meta_copy if raw_meta_copy else raw_meta
def decode_fullpath(self, fullpath):
account, container, path, version, content_id = decode_fullpath(
fullpath)
cid = self.cid_from_name(account, container)
cid, path, version, content_id = self._save_content(
cid, path, version, content_id)
return account, container, cid, path, version, content_id
def chunk_move(self, path, chunk_id):
meta = self.load_chunk_metadata(path, chunk_id)
container_id = meta['container_id']
content_id = meta['content_id']
chunk_id = meta['chunk_id']
try:
content = self.content_factory.get(container_id, content_id)
except ContentNotFound:
raise exc.OrphanChunk('Content not found')
new_chunk = content.move_chunk(chunk_id)
self.logger.info('moved chunk http://%s/%s to %s', self.address,
chunk_id, new_chunk['url'])
if self.allow_links:
old_links = meta['links']
for chunk_id, fullpath in old_links.iteritems():
account, container, _, _, content_id = \
decode_fullpath(fullpath)
container_id = cid_from_name(account, container)
try:
content = self.content_factory.get(container_id,
content_id)
except ContentNotFound:
raise exc.OrphanChunk('Content not found')
new_linked_chunk = content.move_linked_chunk(
chunk_id, new_chunk['url'])
self.logger.info('moved chunk http://%s/%s to %s',
self.address, chunk_id,
new_linked_chunk['url'])
def link(self):
"""
Create new hard links for all the chunks of a metachunk.
"""
new_meta_chunks = list()
failed_chunks = list()
# pylint: disable=unbalanced-tuple-unpacking
acct, ct, path, vers, _ = decode_fullpath(self.fullpath)
cid = cid_from_name(acct, ct)
for chunk_target in self.meta_chunk_target:
try:
chunk_id = compute_chunk_id(cid, path, vers,
chunk_target['pos'], self.policy)
resp, new_chunk_url = self.blob_client.chunk_link(
chunk_target['url'],
chunk_id,
self.fullpath,
connection_timeout=self.connection_timeout,
write_timeout=self.write_timeout,
reqid=self.reqid,
perfdata=self.perfdata,
logger=self.logger)
new_chunk = chunk_target.copy()
new_chunk['url'] = new_chunk_url
new_meta_chunks.append(new_chunk)
except Exception:
failed_chunks.append(chunk_target)
try:
self.quorum_or_fail(new_meta_chunks, failed_chunks)
except Exception as ex:
raise exc.UnfinishedUploadException(ex, new_meta_chunks)
return new_meta_chunks
def decode_fullpath(self, fullpath):
# pylint: disable=unbalanced-tuple-unpacking
account, container, path, version, content_id = decode_fullpath(
fullpath)
cid = self.cid_from_name(account, container)
cid, path, version, content_id = self._save_content(
cid, path, version, content_id)
return account, container, cid, path, version, content_id
def decode_old_fullpath(self, old_fullpath):
# pylint: disable=unbalanced-tuple-unpacking
try:
account, container, path, version = decode_old_fullpath(
old_fullpath)
cid = self.cid_from_name(account, container)
content_id = self.content_id_from_name(cid, path, version)
except ValueError:
# We never know, let's try to decode the fullpath as if it was new
account, container, path, version, content_id = decode_fullpath(
old_fullpath)
cid = self.cid_from_name(account, container)
return account, container, cid, path, version, content_id
def chunk_move(self, path, chunk_id):
meta = self.load_chunk_metadata(path, chunk_id)
container_id = meta['container_id']
content_id = meta['content_id']
chunk_id = meta['chunk_id']
# Maybe skip the chunk because it doesn't match the size constaint
chunk_size = int(meta['chunk_size'])
min_chunk_size = int(self.conf.get('min_chunk_size', 0))
max_chunk_size = int(self.conf.get('max_chunk_size', 0))
if chunk_size < min_chunk_size:
self.logger.debug("SKIP %s too small", path)
return
if max_chunk_size > 0 and chunk_size > max_chunk_size:
self.logger.debug("SKIP %s too big", path)
return
# Start moving the chunk
try:
content = self.content_factory.get(container_id, content_id)
except ContentNotFound:
raise exc.OrphanChunk('Content not found')
new_chunk = content.move_chunk(
chunk_id,
service_id=self.service_id,
fake_excluded_chunks=self.fake_excluded_chunks)
self.logger.info('moved chunk http://%s/%s to %s', self.service_id,
chunk_id, new_chunk['url'])
if self.allow_links:
old_links = meta['links']
for chunk_id, fullpath in old_links.items():
# pylint: disable=unbalanced-tuple-unpacking
account, container, _, _, content_id = \
decode_fullpath(fullpath)
container_id = cid_from_name(account, container)
try:
content = self.content_factory.get(container_id,
content_id)
except ContentNotFound:
raise exc.OrphanChunk('Content not found')
new_linked_chunk = content.move_linked_chunk(
chunk_id, new_chunk['url'])
self.logger.info('moved chunk http://%s/%s to %s',
self.service_id, chunk_id,
new_linked_chunk['url'])
def read_chunk_metadata(fd, chunk_id, check_chunk_id=True):
chunk_id = chunk_id.upper()
raw_meta = read_user_xattr(fd)
raw_meta_copy = None
meta = {}
meta['links'] = dict()
attr_vers = 0.0
raw_chunk_id = container_id = path = version = content_id = None
missing = list()
for k, v in raw_meta.iteritems():
# New chunks have a version
if k == chunk_xattr_keys['oio_version']:
attr_vers = float(v)
# Chunks with version >= 4.2 have a "full_path"
elif k.startswith(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX):
parsed_chunk_id = k[len(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX):]
if parsed_chunk_id == chunk_id:
raw_chunk_id = parsed_chunk_id
meta['full_path'] = v
account, container, path, version, content_id = \
decode_fullpath(v)
container_id = cid_from_name(account, container)
else:
meta['links'][parsed_chunk_id] = v
if raw_chunk_id:
raw_meta_copy = raw_meta.copy()
raw_meta[chunk_xattr_keys['chunk_id']] = raw_chunk_id
raw_meta[chunk_xattr_keys['container_id']] = container_id
raw_meta[chunk_xattr_keys['content_path']] = path
raw_meta[chunk_xattr_keys['content_version']] = version
raw_meta[chunk_xattr_keys['content_id']] = content_id
if attr_vers >= 4.2 and 'full_path' not in meta:
# TODO(FVE): in that case, do not warn about other attributes
# that could be deduced from this one.
missing.append(
exc.MissingAttribute(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX +
chunk_id))
for k, v in chunk_xattr_keys.iteritems():
if v not in raw_meta:
if k not in chunk_xattr_keys_optional:
missing.append(exc.MissingAttribute(v))
else:
meta[k] = raw_meta[v]
if missing:
raise exc.FaultyChunk(*missing)
if check_chunk_id and meta['chunk_id'] != chunk_id:
raise exc.MissingAttribute(chunk_xattr_keys['chunk_id'])
return meta, raw_meta_copy if raw_meta_copy else raw_meta
def complete_target_from_chunk_metadata(self, target, xattr_meta):
"""
Complete a Target object from metadata found in chunk's extended
attributes. In case the "fullpath" is not available, try to read
legacy metadata, and maybe ask meta1 to resolve the CID into
account and container names.
"""
# pylint: disable=unbalanced-tuple-unpacking
try:
acct, ct, path, vers, content_id = \
decode_fullpath(xattr_meta['full_path'])
target.account = acct
target.container = ct
target.obj = path
target.content_id = content_id
target.version = vers
except KeyError:
# No fullpath header, try legacy headers
if 'content_path' in xattr_meta:
target.obj = xattr_meta['content_path']
if 'content_id' in xattr_meta:
target.content_id = xattr_meta['content_id']
if 'content_version' in xattr_meta:
target.version = xattr_meta['content_version']
cid = xattr_meta.get('container_id')
if cid:
try:
md = self.api.directory.show(cid=cid)
acct = md.get('account')
ct = md.get('name')
if acct:
target.account = acct
if ct:
target.container = ct
except Exception as err:
self.logger.warn(
"Failed to resolve CID %s into account "
"and container names: %s", cid, err)
def test_decode_with_utf8_info(self):
info = decode_fullpath(
"m%C5%B7account/mycontain%C3%A9r/my%C3%B6bject/9876543210/"
"0123456789ABCDEF")
self.assertEqual(info, ("mŷaccount", "mycontainér", "myöbject",
"9876543210", "0123456789ABCDEF"))
def test_decode(self):
info = decode_fullpath(
"myaccount/mycontainer/myobject/9876543210/0123456789ABCDEF")
self.assertEqual(info, ("myaccount", "mycontainer", "myobject",
"9876543210", "0123456789ABCDEF"))
