def test_converter_old_chunk_with_link_on_same_object(self):
for c in self.chunks:
convert_to_old_chunk(
self._chunk_path(c), self.account, self.container, self.path,
self.version, self.content_id)
self.api.object_link(
self.account, self.container, self.path,
self.account, self.container, self.path)
linked_meta, linked_chunks = self.api.object_locate(
self.account, self.container, self.path)
self.assertNotEqual(self.content_id, linked_meta['id'])
linked_chunk = random.choice(linked_chunks)
linked_chunk_volume = linked_chunk['url'].split('/')[2]
linked_chunk_id = linked_chunk['url'].split('/')[3]
linked_chunk_path = self._chunk_path(linked_chunk)
# old xattr not removed
_, expected_raw_meta = read_chunk_metadata(linked_chunk_path,
linked_chunk_id)
expected_raw_meta[chunk_xattr_keys['oio_version']] = OIO_VERSION
self._convert_and_check(
linked_chunk_volume, linked_chunk_path,
{linked_chunk_id: (self.account, self.container,
self.path, linked_meta['version'],
linked_meta['id'])},
expected_raw_meta=expected_raw_meta, expected_errors=1)
def chunk_audit(self, path):
with open(path) as f:
try:
meta = read_chunk_metadata(f)
except exc.MissingAttribute as e:
raise exc.FaultyChunk(
'Missing extended attribute %s' % e)
size = int(meta['chunk_size'])
md5_checksum = meta['chunk_hash'].lower()
reader = ChunkReader(f, size, md5_checksum)
with closing(reader):
for buf in reader:
buf_len = len(buf)
self.bytes_running_time = ratelimit(
self.bytes_running_time,
self.max_bytes_per_second,
increment=buf_len)
self.bytes_processed += buf_len
self.total_bytes_processed += buf_len
try:
content_cid = meta['content_cid']
content_path = meta['content_path']
content_attr, data = self.container_client.content_show(
cid=content_cid, path=content_path)
# Check chunk data
chunks_nb = 0
chunk_data = None
for c in data:
if c['url'].endswith(meta['chunk_id']):
chunks_nb += 1 # FIXME: won't work with DUP
chunk_data = c
if not chunk_data:
raise exc.OrphanChunk('Not found in content')
if chunk_data['size'] != int(meta['chunk_size']):
raise exc.FaultyChunk('Invalid chunk size found')
if chunk_data['hash'] != meta['chunk_hash']:
raise exc.FaultyChunk('Invalid chunk hash found')
if chunk_data['pos'] != meta['chunk_pos']:
raise exc.FaultyChunk('Invalid chunk position found')
# Check content data
if content_attr['length'] != meta['content_size']:
raise exc.FaultyChunk('Invalid content size found')
if chunks_nb != int(meta['content_chunksnb']):
self.logger.warn('Invalid number of chunks found')
# TODO: really count chunks and enable the exception
# raise exc.FaultyChunk('Invalid number of chunks found')
except exc.NotFound:
raise exc.OrphanChunk('Chunk not found in container')
def update_index(self, path):
with open(path) as f:
try:
meta = read_chunk_metadata(f)
except exc.MissingAttribute as e:
raise exc.FaultyChunk('Missing extended attribute %s' % e)
data = {'mtime': int(time.time())}
self.index_client.chunk_push(self.volume_id, meta['container_id'],
meta['content_id'], meta['chunk_id'],
**data)
def pass_without_lock(self):
last_report = now()
count, success, fail = 0, 0, 0
if self.namespace != self.volume_ns:
self.logger.warn("Forcing the NS to [%s] (previously [%s])",
self.namespace, self.volume_ns)
self.logger.info("START %s", self.volume)
paths = paths_gen(self.volume)
for path in paths:
chunk_id = path.rsplit('/', 1)[-1]
if len(chunk_id) != STRLEN_CHUNKID:
self.logger.warn('WARN Not a chunk %s' % path)
return
for c in chunk_id:
if c not in hexdigits:
self.logger.warn('WARN Not a chunk %s' % path)
return
# Action
try:
with open(path) as f:
meta = read_chunk_metadata(f, chunk_id)
self.action(self, path, f, meta)
success = success + 1
except NotFound as e:
fail = fail + 1
self.logger.info("ORPHAN %s/%s in %s/%s %s",
meta['content_id'], meta['chunk_id'],
meta['container_id'], meta['content_path'],
str(e))
except Conflict as e:
fail = fail + 1
self.logger.info("ALREADY %s/%s in %s/%s %s",
meta['content_id'], meta['chunk_id'],
meta['container_id'], meta['content_path'],
str(e))
except Exception as e:
fail = fail + 1
self.logger.warn("ERROR %s/%s in %s/%s %s",
meta['content_id'], meta['chunk_id'],
meta['container_id'], meta['content_path'],
str(e))
count = count + 1
# TODO(jfs): do the throttling
# periodical reporting
t = now()
if t - last_report > self.report_interval:
self.logger.info("STEP %d ok %d ko %d",
count, success, fail)
self.logger.info("FINAL %s %d ok %d ko %d",
self.volume, count, success, fail)
def _convert_and_check(self,
chunk_volume,
chunk_path,
chunk_id_info,
expected_raw_meta=None,
expected_errors=0):
conf = self.conf
conf['volume'] = self.rawx_volumes[chunk_volume]
converter = BlobConverter(conf, logger=self.logger)
converter.safe_convert_chunk(chunk_path)
self.assertEqual(1, converter.total_chunks_processed)
self.assertEqual(1, converter.passes)
self.assertEqual(expected_errors, converter.errors)
checker = Checker(self.ns)
for chunk_id, info in chunk_id_info.items():
account, container, path, version, content_id = info
fullpath = encode_fullpath(account, container, path, version,
content_id)
cid = cid_from_name(account, container)
meta, raw_meta = read_chunk_metadata(chunk_path, chunk_id)
self.assertEqual(meta.get('chunk_id'), chunk_id)
self.assertEqual(meta.get('container_id'), cid)
self.assertEqual(meta.get('content_path'), path)
self.assertEqual(meta.get('content_version'), version)
self.assertEqual(meta.get('content_id'), content_id)
self.assertEqual(meta.get('full_path'), fullpath)
checker.check(
Target(account,
container=container,
obj=path,
chunk='http://' + converter.volume_id + '/' + chunk_id))
for _ in checker.run():
pass
self.assertTrue(checker.report())
if expected_raw_meta:
self.assertDictEqual(expected_raw_meta, raw_meta)
continue
self.assertNotIn(CHUNK_XATTR_KEYS['chunk_id'], raw_meta)
self.assertNotIn(CHUNK_XATTR_KEYS['container_id'], raw_meta)
self.assertNotIn(CHUNK_XATTR_KEYS['content_path'], raw_meta)
self.assertNotIn(CHUNK_XATTR_KEYS['content_version'], raw_meta)
self.assertNotIn(CHUNK_XATTR_KEYS['content_id'], raw_meta)
self.assertIn(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + chunk_id,
raw_meta)
for k in raw_meta.keys():
if k.startswith('oio:'):
self.fail('old fullpath always existing')
self.assertEqual(raw_meta[CHUNK_XATTR_KEYS['oio_version']],
OIO_VERSION)
def update_index(self, path):
with open(path) as f:
try:
meta = read_chunk_metadata(f)
except exc.MissingAttribute as e:
raise exc.FaultyChunk(
'Missing extended attribute %s' % e)
data = {'mtime': int(time.time())}
self.index_client.chunk_push(self.volume_id,
meta['container_id'],
meta['content_id'],
meta['chunk_id'],
**data)
def chunk_file_audit(self, chunk_file, chunk_id):
try:
meta, _ = read_chunk_metadata(chunk_file, chunk_id)
except exc.MissingAttribute as err:
raise exc.FaultyChunk(err)
size = int(meta['chunk_size'])
md5_checksum = meta['chunk_hash'].lower()
reader = ChunkReader(chunk_file,
size,
md5_checksum,
compression=meta.get("compression", ""))
with closing(reader):
for buf in reader:
buf_len = len(buf)
self.bytes_running_time = ratelimit(self.bytes_running_time,
self.max_bytes_per_second,
increment=buf_len)
self.bytes_processed += buf_len
self.total_bytes_processed += buf_len
try:
container_id = meta['container_id']
content_id = meta['content_id']
_obj_meta, data = self.container_client.content_locate(
cid=container_id, content=content_id, properties=False)
# Check chunk data
chunk_data = None
metachunks = set()
for c in data:
if c['url'].endswith(meta['chunk_id']):
metachunks.add(c['pos'].split('.', 2)[0])
chunk_data = c
if not chunk_data:
raise exc.OrphanChunk('Not found in content')
metachunk_size = meta.get('metachunk_size')
if metachunk_size is not None \
and chunk_data['size'] != int(metachunk_size):
raise exc.FaultyChunk('Invalid metachunk size found')
metachunk_hash = meta.get('metachunk_hash')
if metachunk_hash is not None \
and chunk_data['hash'] != meta['metachunk_hash']:
raise exc.FaultyChunk('Invalid metachunk hash found')
if chunk_data['pos'] != meta['chunk_pos']:
raise exc.FaultyChunk('Invalid chunk position found')
except exc.NotFound:
raise exc.OrphanChunk('Chunk not found in container')
def chunk_audit(self, path):
with open(path) as f:
try:
meta = read_chunk_metadata(f)
except exc.MissingAttribute as e:
raise exc.FaultyChunk('Missing extended attribute %s' % e)
size = int(meta['chunk_size'])
md5_checksum = meta['chunk_hash'].lower()
reader = ChunkReader(f, size, md5_checksum)
with closing(reader):
for buf in reader:
buf_len = len(buf)
self.bytes_running_time = ratelimit(
self.bytes_running_time,
self.max_bytes_per_second,
increment=buf_len)
self.bytes_processed += buf_len
self.total_bytes_processed += buf_len
try:
container_id = meta['container_id']
content_path = meta['content_path']
content_attr, data = self.container_client.content_show(
cid=container_id, path=content_path)
# Check chunk data
chunk_data = None
metachunks = set()
for c in data:
if c['url'].endswith(meta['chunk_id']):
metachunks.add(c['pos'].split('.', 2)[0])
chunk_data = c
if not chunk_data:
raise exc.OrphanChunk('Not found in content')
if chunk_data['size'] != int(meta['chunk_size']):
raise exc.FaultyChunk('Invalid chunk size found')
if chunk_data['hash'] != meta['chunk_hash']:
raise exc.FaultyChunk('Invalid chunk hash found')
if chunk_data['pos'] != meta['chunk_pos']:
raise exc.FaultyChunk('Invalid chunk position found')
except exc.NotFound:
raise exc.OrphanChunk('Chunk not found in container')
def update_index(self, path, chunk_id):
with open(path) as file_:
try:
meta = None
if meta is None:
meta, _ = read_chunk_metadata(file_, chunk_id)
except exc.MissingAttribute as err:
raise exc.FaultyChunk(err)
data = {'mtime': int(time.time())}
headers = {REQID_HEADER: request_id('blob-indexer-')}
self.index_client.chunk_push(self.volume_id,
meta['container_id'],
meta['content_id'],
meta['chunk_id'],
headers=headers,
**data)
def chunk_audit(self, path):
with open(path) as f:
try:
meta = read_chunk_metadata(f)
except exc.MissingAttribute as e:
raise exc.FaultyChunk("Missing extended attribute %s" % e)
size = int(meta["chunk_size"])
md5_checksum = meta["chunk_hash"].lower()
reader = ChunkReader(f, size, md5_checksum)
with closing(reader):
for buf in reader:
buf_len = len(buf)
self.bytes_running_time = ratelimit(
self.bytes_running_time, self.max_bytes_per_second, increment=buf_len
)
self.bytes_processed += buf_len
self.total_bytes_processed += buf_len
try:
container_id = meta["container_id"]
content_path = meta["content_path"]
content_attr, data = self.container_client.content_show(cid=container_id, path=content_path)
# Check chunk data
chunk_data = None
metachunks = set()
for c in data:
if c["url"].endswith(meta["chunk_id"]):
metachunks.add(c["pos"].split(".", 2)[0])
chunk_data = c
if not chunk_data:
raise exc.OrphanChunk("Not found in content")
if chunk_data["size"] != int(meta["chunk_size"]):
raise exc.FaultyChunk("Invalid chunk size found")
if chunk_data["hash"] != meta["chunk_hash"]:
raise exc.FaultyChunk("Invalid chunk hash found")
if chunk_data["pos"] != meta["chunk_pos"]:
raise exc.FaultyChunk("Invalid chunk position found")
except exc.NotFound:
raise exc.OrphanChunk("Chunk not found in container")
def update_index(self, path):
with open(path) as f:
try:
meta = read_chunk_metadata(f)
except exc.MissingAttribute as e:
raise exc.FaultyChunk('Missing extended attribute %s' % e)
data = {
'content_version': meta['content_version'],
'content_nbchunks': meta['content_chunksnb'],
'content_path': meta['content_path'],
'content_size': meta['content_size'],
'chunk_hash': meta['chunk_hash'],
'chunk_position': meta['chunk_pos'],
'chunk_size': meta['chunk_size'],
'mtime': int(time.time())
}
self.index_client.chunk_push(self.volume_id, meta['content_cid'],
meta['content_id'], meta['chunk_id'],
**data)
def update_index(self, path, chunk_id):
with open(path) as f:
try:
meta = None
if self.convert_chunks and self.converter:
_, meta = self.converter.convert_chunk(f, chunk_id)
if meta is None:
meta, _ = read_chunk_metadata(f, chunk_id)
except exc.MissingAttribute as e:
raise exc.FaultyChunk('Missing extended attribute %s' % e)
data = {'mtime': int(time.time())}
headers = {'X-oio-req-id': 'blob-indexer-' + request_id()[:-13]}
self.index_client.chunk_push(self.volume_id,
meta['container_id'],
meta['content_id'],
meta['chunk_id'],
headers=headers,
**data)
def chunk_audit(self, path):
with open(path) as f:
try:
meta = read_chunk_metadata(f)
except exc.MissingAttribute as e:
raise exc.FaultyChunk(
'Missing extended attribute %s' % e)
size = int(meta['chunk_size'])
md5_checksum = meta['chunk_hash'].lower()
reader = ChunkReader(f, size, md5_checksum)
with closing(reader):
for buf in reader:
buf_len = len(buf)
self.bytes_running_time = ratelimit(
self.bytes_running_time,
self.max_bytes_per_second,
increment=buf_len)
self.bytes_processed += buf_len
self.total_bytes_processed += buf_len
try:
content_cid = meta['content_cid']
content_path = meta['content_path']
_, data = self.container_client.content_show(
cid=content_cid, path=content_path)
chunk_data = None
for c in data:
if c['url'].endswith(meta['chunk_id']):
chunk_data = c
if not chunk_data:
raise exc.OrphanChunk('Not found in content')
if chunk_data['size'] != int(meta['chunk_size']):
raise exc.FaultyChunk('Invalid chunk size found')
if chunk_data['hash'] != meta['chunk_hash']:
raise exc.FaultyChunk('Invalid chunk hash found')
if chunk_data['pos'] != meta['chunk_pos']:
raise exc.FaultyChunk('Invalid chunk position found')
except exc.NotFound:
raise exc.OrphanChunk('Chunk not found in container')
def pass_chunk_file(self, path):
chunk_id = path.rsplit('/', 1)[-1]
if len(chunk_id) != STRLEN_CHUNKID:
if chunk_id.endswith(CHUNK_SUFFIX_PENDING):
self.logger.info('Skipping pending chunk %s', path)
else:
self.logger.warn('WARN Not a chunk %s', path)
return
for char in chunk_id:
if char not in hexdigits:
self.logger.warn('WARN Not a chunk %s', path)
return
with open(path) as f:
meta, _ = read_chunk_metadata(f, chunk_id)
if self.container_ids \
and meta['container_id'] in self.container_ids:
self.logger.debug(
'Skipping chunk file (container_id=%s content_path=%s '
'content_version=%s content_id=%s chunk_id=%s '
'chunk_pos=%s)', meta['container_id'],
meta['content_path'], meta['content_version'],
meta['content_id'], meta['chunk_id'], meta['chunk_pos'])
return
beans = self._beans_from_meta(meta)
for bean in beans:
try:
self.pass_bean(meta, bean)
except Exception as exc:
self.logger.error(
'Failed to pass chunk file (container_id=%s '
'content_path=%s content_version=%s content_id=%s '
'chunk_id=%s chunk_pos=%s): %s', meta['container_id'],
meta['content_path'], meta['content_version'],
meta['content_id'], meta['chunk_id'],
meta['chunk_pos'], exc)
self.bean_errors[bean['type']] = \
self.bean_errors[bean['type']] + 1
def update_index(self, path):
with open(path) as f:
try:
meta = read_chunk_metadata(f)
except exc.MissingAttribute as e:
raise exc.FaultyChunk(
'Missing extended attribute %s' % e)
data = {
'content_version': meta['content_version'],
'content_nbchunks': meta['content_chunksnb'],
'content_path': meta['content_path'],
'content_size': meta['content_size'],
'chunk_hash': meta['chunk_hash'],
'chunk_position': meta['chunk_pos'],
'chunk_size': meta['chunk_size'],
'mtime': int(time.time())
}
self.index_client.chunk_push(self.volume_id,
meta['content_cid'],
meta['content_id'],
meta['chunk_id'],
**data)
def convert_chunk(self, fd, chunk_id):
meta, raw_meta = read_chunk_metadata(fd,
chunk_id,
check_chunk_id=False)
links = meta.get('links', dict())
for chunk_id2, fullpath2 in links.iteritems():
self.decode_fullpath(fullpath2)
fullpath = meta.get('full_path')
if fullpath is not None:
self.decode_fullpath(fullpath)
if meta.get('oio_version') == OIO_VERSION:
return True, meta
chunk_inode = os.fstat(fd.fileno()).st_ino
raw_chunk_id = None
chunk_id = chunk_id.upper()
chunk_pos = meta['chunk_pos']
container_id = meta['container_id'].upper()
path = meta['content_path']
version = meta['content_version']
content_id = meta['content_id'].upper()
new_fullpaths = dict()
xattr_to_remove = list()
success = True
for k, v in raw_meta.iteritems():
# fetch raw chunk ID
if k == XATTR_CHUNK_ID:
raw_chunk_id = v.upper()
# search old fullpaths
if not k.startswith(XATTR_OLD_FULLPATH) \
or not is_hexa(k[4:], size=64):
continue
try:
account2, container2, container_id2, path2, version2, \
content_id2 = self.decode_old_fullpath(v)
if container_id == container_id2 and path == path2 \
and version == version2:
if content_id2 is None:
content_id2 = self.content_id_from_name(container_id2,
path2,
version2,
search=True)
chunk_id, new_fullpath = self.encode_fullpath(
chunk_inode, chunk_id, account2, container2, path2,
version2, content_id2)
new_fullpaths[chunk_id] = new_fullpath
else:
chunk_id2, new_fullpath = self.get_chunk_id_and_fullpath(
chunk_inode,
chunk_pos,
container_id2,
path2,
version2,
account=account2,
container=container2,
content_id=content_id2)
new_fullpaths[chunk_id2] = new_fullpath
xattr_to_remove.append(k)
except Exception as exc:
success = False
self.logger.warn('chunk_id=%s old_fullpath=%s: %s', chunk_id,
k, exc)
# old xattr
if raw_chunk_id is not None:
try:
if raw_chunk_id != chunk_id and raw_chunk_id not in links:
if raw_chunk_id not in new_fullpaths:
meta2, _ = read_chunk_metadata(fd, raw_chunk_id)
container_id2 = meta2['container_id'].upper()
path2 = meta2['content_path']
version2 = meta2['content_version']
content_id2 = meta2['content_id'].upper()
raw_chunk_id, new_fullpath = \
self.get_chunk_id_and_fullpath(
chunk_inode, chunk_pos, container_id2, path2,
version2, chunk_id=raw_chunk_id,
content_id=content_id2)
new_fullpaths[raw_chunk_id] = new_fullpath
elif raw_chunk_id == chunk_id and fullpath is None:
if raw_chunk_id not in new_fullpaths:
raw_chunk_id, new_fullpath = \
self.get_chunk_id_and_fullpath(
chunk_inode, chunk_pos, container_id, path,
version, chunk_id=raw_chunk_id,
content_id=content_id)
new_fullpaths[raw_chunk_id] = new_fullpath
except Exception as exc:
success = False
self.logger.warn('chunk_id=%s (old xattr): %s', raw_chunk_id,
exc)
self.save_xattr(chunk_id, raw_meta)
if self.dry_run:
self.logger.info(
"[dryrun] Converting chunk %s: success=%s new_fullpaths=%s "
"xattr_to_remove=%s", chunk_id, str(success),
str(new_fullpaths), str(xattr_to_remove))
else:
# for security, if there is an error, we don't delete old xattr
modify_xattr(fd, new_fullpaths, success, xattr_to_remove)
return success, None
def _cycle_copy(self, path):
if path:
self.path = path
chunkid = random_chunk_id()
chunkdata = random_buffer(string.printable, 1).encode('utf-8')
chunkurl = self._rawx_url(chunkid)
chunkpath = self._chunk_path(chunkid)
headers1 = self._chunk_attr(chunkid, chunkdata)
metachunk_hash = md5().hexdigest()
trailers = {'x-oio-chunk-meta-metachunk-size': '1',
'x-oio-chunk-meta-metachunk-hash': metachunk_hash}
self._check_not_present(chunkurl)
resp, _ = self._http_request(chunkurl, 'PUT', chunkdata, headers1,
trailers)
self.assertEqual(201, resp.status)
self.assertEqual(headers1['x-oio-chunk-meta-chunk-hash'].upper(),
resp.getheader('x-oio-chunk-meta-chunk-hash'))
self.assertEqual(headers1['x-oio-chunk-meta-chunk-size'],
resp.getheader('x-oio-chunk-meta-chunk-size'))
copyid = random_chunk_id()
copyid = chunkid[:-60] + copyid[-60:]
copyurl = self._rawx_url(copyid)
copypath = self._chunk_path(copyid)
headers2 = {}
headers2["Destination"] = copyurl
copy_account = "account-snapshot"
copy_container = "container-snapshot"
copy_container_id = cid_from_name(copy_account, copy_container)
copy_path = path+"-snapshot"
copy_version = 1456938361143741
copy_id = random_id(32)
copy_fullpath = encode_fullpath(
copy_account, copy_container, copy_path, copy_version, copy_id)
headers2['x-oio-chunk-meta-full-path'] = copy_fullpath
resp, _ = self._http_request(chunkurl, 'COPY', '', headers2)
self.assertEqual(201, resp.status)
resp, body = self._http_request(chunkurl, 'GET', '', {})
self.assertEqual(200, resp.status)
headers1['x-oio-chunk-meta-chunk-hash'] = \
headers1['x-oio-chunk-meta-chunk-hash'].upper()
for k, v in headers1.items():
if k == 'x-oio-chunk-meta-content-path':
self.assertEqual(unquote(resp.getheader(k)), unquote(str(v)))
else:
self.assertEqual(resp.getheader(k), str(v))
resp, body = self._http_request(copyurl, 'GET', '', {})
self.assertEqual(200, resp.status)
headers2_bis = headers1.copy()
headers2_bis['x-oio-chunk-meta-full-path'] = \
headers2['x-oio-chunk-meta-full-path']
headers2_bis['x-oio-chunk-meta-content-path'] = copy_path
headers2_bis['x-oio-chunk-meta-content-version'] = copy_version
headers2_bis['x-oio-chunk-meta-content-id'] = copy_id
headers2_bis['x-oio-chunk-meta-container-id'] = copy_container_id
headers2_bis['x-oio-chunk-meta-chunk-id'] = copyid
for k, v in headers2_bis.items():
if k == 'x-oio-chunk-meta-content-path':
self.assertEqual(unquote(resp.getheader(k)), unquote(str(v)))
else:
self.assertEqual(resp.getheader(k), str(v))
with open(chunkpath, 'r') as fd:
meta, _ = read_chunk_metadata(fd, chunkid)
self.assertEqual(headers1['x-oio-chunk-meta-full-path'],
meta['full_path'])
self.assertEqual(1, len(meta['links']))
self.assertEqual(headers2['x-oio-chunk-meta-full-path'],
meta['links'][copyid])
with open(copypath, 'r') as fd:
meta, _ = read_chunk_metadata(fd, copyid)
self.assertEqual(headers2['x-oio-chunk-meta-full-path'],
meta['full_path'])
self.assertEqual(1, len(meta['links']))
self.assertEqual(headers1['x-oio-chunk-meta-full-path'],
meta['links'][chunkid])
resp, body = self._http_request(chunkurl, 'DELETE', '', {})
self.assertEqual(204, resp.status)
resp, body = self._http_request(chunkurl, 'GET', '', {})
self.assertEqual(404, resp.status)
resp, body = self._http_request(copyurl, 'GET', '', {})
self.assertEqual(200, resp.status)
self.assertEqual(headers2['x-oio-chunk-meta-full-path'],
resp.getheader('x-oio-chunk-meta-full-path'))
with open(copypath, 'r') as fd:
meta, _ = read_chunk_metadata(fd, copyid)
self.assertEqual(headers2['x-oio-chunk-meta-full-path'],
meta['full_path'])
self.assertEqual(0, len(meta['links']))
resp, body = self._http_request(copyurl, 'DELETE', '', {})
self.assertEqual(204, resp.status)
resp, body = self._http_request(copyurl, 'GET', '', {})
self.assertEqual(404, resp.status)
def test_read_old_chunk(self):
metachunk_hash = md5().hexdigest()
trailers = {'x-oio-chunk-meta-metachunk-size': '1',
'x-oio-chunk-meta-metachunk-hash': metachunk_hash}
chunkid = random_chunk_id()
chunkdata = random_buffer(string.printable, 1).encode('utf-8')
chunkurl = self._rawx_url(chunkid)
chunkpath = self._chunk_path(chunkid)
headers = self._chunk_attr(chunkid, chunkdata)
self._check_not_present(chunkurl)
resp, _ = self._http_request(chunkurl, 'PUT', chunkdata, headers,
trailers)
self.assertEqual(201, resp.status)
resp1, data1 = self._http_request(chunkurl, 'GET', '', {})
self.assertEqual(200, resp1.status)
headers1 = HeadersDict(resp1.getheaders())
with open(chunkpath, 'r') as fd:
meta1, _ = read_chunk_metadata(fd, chunkid)
convert_to_old_chunk(
chunkpath, self.account, self.container, self.content_path,
self.content_version, self.content_id)
resp2, data2 = self._http_request(chunkurl, 'GET', '', {})
self.assertEqual(200, resp2.status)
headers2 = HeadersDict(resp2.getheaders())
with open(chunkpath, 'r') as fd:
meta2, _ = read_chunk_metadata(fd, chunkid)
self.assertEqual(data1, data2)
del headers1[CHUNK_HEADERS['full_path']]
del headers1[CHUNK_HEADERS['oio_version']]
del headers2[CHUNK_HEADERS['oio_version']]
del headers1["date"]
del headers2["date"]
self.assertDictEqual(headers1, headers2)
del meta1['full_path']
del meta1['oio_version']
del meta2['oio_version']
self.assertDictEqual(meta1, meta2)
# Copy old chunk
copyid = random_chunk_id()
copyid = chunkid[:-60] + copyid[-60:]
copyurl = self._rawx_url(copyid)
copypath = self._chunk_path(copyid)
copycontentid = random_id(32)
copyheaders = {}
copyheaders["Destination"] = copyurl
copyheaders['x-oio-chunk-meta-full-path'] = encode_fullpath(
"account-snapshot", "container-snapshot",
self.content_path+"-snapshot", 1456938361143741, copycontentid)
resp, _ = self._http_request(chunkurl, 'COPY', '', copyheaders)
self.assertEqual(201, resp.status)
resp2, data2 = self._http_request(chunkurl, 'GET', '', {})
self.assertEqual(200, resp2.status)
headers2 = HeadersDict(resp2.getheaders())
with open(chunkpath, 'r') as fd:
meta2, _ = read_chunk_metadata(fd, chunkid)
self.assertEqual(1, len(meta2['links']))
self.assertEqual(copyheaders['x-oio-chunk-meta-full-path'],
meta2['links'][copyid])
meta2['links'] = dict()
self.assertEqual(data1, data2)
del headers2[CHUNK_HEADERS['oio_version']]
del headers2["date"]
self.assertDictEqual(headers1, headers2)
del meta2['oio_version']
self.assertDictEqual(meta1, meta2)
resp3, data3 = self._http_request(copyurl, 'GET', '', {})
self.assertEqual(200, resp3.status)
headers3 = HeadersDict(resp3.getheaders())
with open(copypath, 'r') as fd:
meta3, _ = read_chunk_metadata(fd, copyid)
self.assertEqual(
copyheaders['x-oio-chunk-meta-full-path'],
headers3['x-oio-chunk-meta-full-path'])
del headers3['x-oio-chunk-meta-full-path']
self.assertEqual(
cid_from_name("account-snapshot", "container-snapshot"),
headers3['x-oio-chunk-meta-container-id'])
del headers1['x-oio-chunk-meta-container-id']
del headers3['x-oio-chunk-meta-container-id']
self.assertEqual(
self.content_path+"-snapshot",
unquote(headers3['x-oio-chunk-meta-content-path']))
del headers1['x-oio-chunk-meta-content-path']
del headers3['x-oio-chunk-meta-content-path']
self.assertEqual(
'1456938361143741',
headers3['x-oio-chunk-meta-content-version'])
del headers1['x-oio-chunk-meta-content-version']
del headers3['x-oio-chunk-meta-content-version']
self.assertEqual(
copycontentid, headers3['x-oio-chunk-meta-content-id'])
del headers1['x-oio-chunk-meta-content-id']
del headers3['x-oio-chunk-meta-content-id']
self.assertEqual(copyid, headers3['x-oio-chunk-meta-chunk-id'])
del headers1['x-oio-chunk-meta-chunk-id']
del headers3['x-oio-chunk-meta-chunk-id']
self.assertEqual(
copyheaders['x-oio-chunk-meta-full-path'], meta3['full_path'])
del meta3['full_path']
self.assertEqual(
cid_from_name("account-snapshot", "container-snapshot"),
meta3['container_id'])
del meta1['container_id']
del meta3['container_id']
self.assertEqual(self.content_path+"-snapshot", meta3['content_path'])
del meta1['content_path']
del meta3['content_path']
self.assertEqual('1456938361143741', meta3['content_version'])
del meta1['content_version']
del meta3['content_version']
self.assertEqual(copycontentid, meta3['content_id'])
del meta1['content_id']
del meta3['content_id']
self.assertEqual(copyid, meta3['chunk_id'])
del meta1['chunk_id']
del meta3['chunk_id']
# FIXME the old chunk is invisible
self.assertEqual(0, len(meta3['links']))
self.assertEqual(data1, data3)
del headers3[CHUNK_HEADERS['oio_version']]
del headers3["date"]
self.assertDictEqual(headers1, headers3)
del meta3['oio_version']
self.assertDictEqual(meta1, meta3)
def load_chunk_metadata(self, path, chunk_id):
with open(path) as file_:
meta, _ = read_chunk_metadata(file_, chunk_id)
return meta
def load_chunk_metadata(self, path):
with open(path) as f:
return read_chunk_metadata(f)
def load_chunk_metadata(self, path):
with open(path) as f:
return read_chunk_metadata(f)