def recover_chunk_fullpath(self, path, chunk_id=None):
if not chunk_id:
chunk_id = path.rsplit('/', 1)[-1]
# 1. Fetch chunk list from rdir (could be cached).
# Unfortunately we cannot seek for a chunk ID.
entries = [
x for x in self.rdir.chunk_fetch(self.volume_id, limit=-1)
if x[2] == chunk_id
]
if not entries:
raise KeyError('Chunk %s not found in rdir' % chunk_id)
elif len(entries) > 1:
self.logger.info('Chunk %s appears in %d objects', chunk_id,
len(entries))
# 2. Find content and container IDs
cid, content_id = entries[0][0:2]
# 3a. Call ContainerClient.content_locate()
# with the container ID and content ID
try:
meta, chunks = self.container_client.content_locate(
cid=cid, content=content_id)
except NotFound as err:
raise OrphanChunk('Cannot check %s is valid: %s' % (path, err))
# 3b. Resolve container ID into account and container names.
# FIXME(FVE): get account and container names from meta1
cmeta = self.container_client.container_get_properties(cid=cid)
aname = cmeta['system']['sys.account']
cname = cmeta['system']['sys.user.name']
fullpath = encode_fullpath(aname, cname, meta['name'], meta['version'],
content_id)
# 4. Check if the chunk actually belongs to the object
chunk_url = 'http://%s/%s' % (self.volume_id, chunk_id)
if chunk_url not in [x['url'] for x in chunks]:
raise OrphanChunk('Chunk %s not found in object %s' %
(chunk_url, fullpath))
# 5. Regenerate the fullpath
with open(path, 'w') as fd:
set_fullpath_xattr(fd, {chunk_id: fullpath})
return True
def convert_chunk(self, fd, chunk_id):
meta, raw_meta = read_chunk_metadata(fd, chunk_id, for_conversion=True)
links = meta.get('links', dict())
for chunk_id2, fullpath2 in links.items():
self.decode_fullpath(fullpath2)
fullpath = meta.get('full_path')
if fullpath is not None:
self.decode_fullpath(fullpath)
if meta.get('oio_version') == OIO_VERSION:
return True, meta
raw_chunk_id = None
chunk_id = chunk_id.upper()
chunk_pos = meta['chunk_pos']
container_id = meta['container_id'].upper()
path = meta['content_path']
version = meta['content_version']
content_id = meta['content_id'].upper()
new_fullpaths = dict()
xattr_to_remove = list()
success = True
for k, v in raw_meta.items():
# fetch raw chunk ID
if k == XATTR_CHUNK_ID:
raw_chunk_id = v.upper()
# search old fullpaths
if not k.startswith(XATTR_OLD_FULLPATH) \
or not is_hexa(k[4:], size=64):
continue
try:
account2, container2, container_id2, path2, version2, \
content_id2 = self.decode_old_fullpath(v)
if meta['chunk_id'] == chunk_id \
and container_id == container_id2 \
and path == path2 \
and version == version2:
if content_id2 is None:
content_id2 = self.content_id_from_name(container_id2,
path2,
version2,
search=True)
chunk_id2, new_fullpath = self.encode_fullpath(
fd, chunk_id, account2, container2, path2, version2,
content_id2)
new_fullpaths[chunk_id2] = new_fullpath
else:
chunk_id2, new_fullpath = self.get_chunk_id_and_fullpath(
fd,
chunk_pos,
container_id2,
path2,
version2,
account=account2,
container=container2,
content_id=content_id2)
new_fullpaths[chunk_id2] = new_fullpath
xattr_to_remove.append(k)
except Exception as exc:
success = False
self.logger.warn('chunk_id=%s old_fullpath=%s: %s', chunk_id,
k, exc)
# old xattr
if raw_chunk_id is not None:
try:
if raw_chunk_id != chunk_id and raw_chunk_id not in links:
if raw_chunk_id not in new_fullpaths:
meta2, _ = read_chunk_metadata(fd, raw_chunk_id)
container_id2 = meta2['container_id'].upper()
path2 = meta2['content_path']
version2 = meta2['content_version']
content_id2 = meta2['content_id'].upper()
raw_chunk_id2, new_fullpath = \
self.get_chunk_id_and_fullpath(
fd, chunk_pos, container_id2, path2,
version2, chunk_id=raw_chunk_id,
content_id=content_id2)
new_fullpaths[raw_chunk_id2] = new_fullpath
elif raw_chunk_id == chunk_id and fullpath is None:
if raw_chunk_id not in new_fullpaths:
raw_chunk_id2, new_fullpath = \
self.get_chunk_id_and_fullpath(
fd, chunk_pos, container_id, path,
version, chunk_id=raw_chunk_id,
content_id=content_id)
new_fullpaths[raw_chunk_id2] = new_fullpath
except Exception as exc:
success = False
self.logger.warn('chunk_id=%s (old xattr): %s', raw_chunk_id,
exc)
self.save_xattr(fd, chunk_id, raw_meta)
if self.dry_run:
self.logger.info(
"[dryrun] Converting chunk %s: success=%s new_fullpaths=%s "
"xattr_to_remove=%s", chunk_id, str(success),
str(new_fullpaths), str(xattr_to_remove))
else:
# for security, if there is an error, we don't delete old xattr
set_fullpath_xattr(fd, new_fullpaths, success, xattr_to_remove)
return success, None