def _fill_queue(self, queue, **kwargs):
if self._fill_queue_from_file(queue, **kwargs):
return
prefixes = set()
rawx_services = self.conscience.all_services('rawx')
for rawx in rawx_services:
cid = cid_from_name('_RDIR', rawx['addr'])
prefix = cid[:self.meta1_digits]
if prefix not in prefixes:
queue.put(prefix.ljust(64, '0'))
prefixes.add(prefix)
accounts = self.api.account_list()
for account in accounts:
containers = self._full_container_list(account)
for container in containers:
cid = cid_from_name(account, container[0])
prefix = cid[:self.meta1_digits]
if prefix not in prefixes:
queue.put(prefix.ljust(64, '0'))
prefixes.add(prefix)
if not self.running:
break
def _populate(self):
self.container_0_name = "container_empty"
self.container_0_id = cid_from_name(
self.test_account, self.container_0_name)
self.container_client.container_create(
acct=self.test_account, ref=self.container_0_name)
self.container_1_name = "container_with_1_content"
self.container_1_id = cid_from_name(
self.test_account, self.container_1_name)
self.container_client.container_create(
acct=self.test_account, ref=self.container_1_name)
self.container_1_content_0_name = "container_1_content_0"
self.container_1_content_0 = self._new_content(
self.container_1_id, self.container_1_content_0_name, "SINGLE")
self.container_2_name = "container_with_2_contents"
self.container_2_id = cid_from_name(
self.test_account, self.container_2_name)
self.container_client.container_create(
acct=self.test_account, ref=self.container_2_name)
self.container_2_content_0_name = "container_2_content_0"
self.container_2_content_0 = self._new_content(
self.container_2_id, self.container_2_content_0_name, "SINGLE")
self.container_2_content_1_name = "container_2_content_1"
self.container_2_content_1 = self._new_content(
self.container_2_id, self.container_2_content_1_name, "TWOCOPIES")
def _populate(self):
self.container_0_name = "container_empty"
self.container_0_id = cid_from_name(self.test_account,
self.container_0_name)
self.container_client.container_create(acct=self.test_account,
ref=self.container_0_name)
self.container_1_name = "container_with_1_content"
self.container_1_id = cid_from_name(self.test_account,
self.container_1_name)
self.container_client.container_create(acct=self.test_account,
ref=self.container_1_name)
self.container_1_content_0_name = "container_1_content_0"
self.container_1_content_0 = self._new_content(
self.container_1_id, self.container_1_content_0_name, "SINGLE")
self.container_2_name = "container_with_2_contents"
self.container_2_id = cid_from_name(self.test_account,
self.container_2_name)
self.container_client.container_create(acct=self.test_account,
ref=self.container_2_name)
self.container_2_content_0_name = "container_2_content_0"
self.container_2_content_0 = self._new_content(
self.container_2_id, self.container_2_content_0_name, "SINGLE")
self.container_2_content_1_name = "container_2_content_1"
self.container_2_content_1 = self._new_content(
self.container_2_id, self.container_2_content_1_name, "TWOCOPIES")
def setUp(self):
super(TestMeta2EventsEmission, self).setUp()
self.container_name = 'TestEventsEmission%f' % time.time()
self.container_id = cid_from_name(self.account, self.container_name)
self.container_client = ContainerClient(self.conf)
self.storage_api = ObjectStorageApi(self.conf['namespace'])
self.beanstalkd0.drain_tube('oio-preserved')
def chunk_move(self, path, chunk_id):
meta = self.load_chunk_metadata(path, chunk_id)
container_id = meta['container_id']
content_id = meta['content_id']
chunk_id = meta['chunk_id']
try:
content = self.content_factory.get(container_id, content_id)
except ContentNotFound:
raise exc.OrphanChunk('Content not found')
new_chunk = content.move_chunk(chunk_id)
self.logger.info('moved chunk http://%s/%s to %s', self.address,
chunk_id, new_chunk['url'])
if self.allow_links:
old_links = meta['links']
for chunk_id, fullpath in old_links.iteritems():
account, container, _, _, content_id = \
decode_fullpath(fullpath)
container_id = cid_from_name(account, container)
try:
content = self.content_factory.get(container_id,
content_id)
except ContentNotFound:
raise exc.OrphanChunk('Content not found')
new_linked_chunk = content.move_linked_chunk(
chunk_id, new_chunk['url'])
self.logger.info('moved chunk http://%s/%s to %s',
self.address, chunk_id,
new_linked_chunk['url'])
def read_chunk_metadata(fd, chunk_id, check_chunk_id=True):
chunk_id = chunk_id.upper()
raw_meta = read_user_xattr(fd)
raw_meta_copy = None
meta = {}
meta['links'] = dict()
raw_chunk_id = container_id = path = version = content_id = None
for k, v in raw_meta.iteritems():
# New chunk
if k.startswith(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX):
chunkid = k[len(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX):]
if chunkid == chunk_id:
raw_chunk_id = chunkid
meta['full_path'] = v
account, container, path, version, content_id = \
decode_fullpath(v)
container_id = cid_from_name(account, container)
else:
meta['links'][chunkid] = v
if raw_chunk_id:
raw_meta_copy = raw_meta.copy()
raw_meta[chunk_xattr_keys['chunk_id']] = raw_chunk_id
raw_meta[chunk_xattr_keys['container_id']] = container_id
raw_meta[chunk_xattr_keys['content_path']] = path
raw_meta[chunk_xattr_keys['content_version']] = version
raw_meta[chunk_xattr_keys['content_id']] = content_id
for k, v in chunk_xattr_keys.iteritems():
if v not in raw_meta:
if k not in chunk_xattr_keys_optional:
raise exc.MissingAttribute(v)
else:
meta[k] = raw_meta[v]
if check_chunk_id and meta['chunk_id'] != chunk_id:
raise exc.MissingAttribute(chunk_xattr_keys['chunk_id'])
return meta, raw_meta_copy if raw_meta_copy else raw_meta
def link(self):
"""
Create new hard links for all the chunks of a metachunk.
"""
new_meta_chunks = list()
failed_chunks = list()
# pylint: disable=unbalanced-tuple-unpacking
acct, ct, path, vers, _ = decode_fullpath(self.fullpath)
cid = cid_from_name(acct, ct)
for chunk_target in self.meta_chunk_target:
try:
chunk_id = compute_chunk_id(cid, path, vers,
chunk_target['pos'], self.policy)
resp, new_chunk_url = self.blob_client.chunk_link(
chunk_target['url'],
chunk_id,
self.fullpath,
connection_timeout=self.connection_timeout,
write_timeout=self.write_timeout,
reqid=self.reqid,
perfdata=self.perfdata,
logger=self.logger)
new_chunk = chunk_target.copy()
new_chunk['url'] = new_chunk_url
new_meta_chunks.append(new_chunk)
except Exception:
failed_chunks.append(chunk_target)
try:
self.quorum_or_fail(new_meta_chunks, failed_chunks)
except Exception as ex:
raise exc.UnfinishedUploadException(ex, new_meta_chunks)
return new_meta_chunks
def setUp(self):
super(TestPlainContent, self).setUp()
if len(self.conf['services']['rawx']) < 4:
self.skipTest(
"Plain tests needs more than 3 rawx to run")
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.chunk_size = self.conf['chunk_size']
self.gridconf = {"namespace": self.namespace}
self.content_factory = ContentFactory(
self.gridconf, logger=self.logger)
self.container_client = ContainerClient(
self.gridconf, logger=self.logger)
self.blob_client = BlobClient(self.conf, logger=self.logger)
self.container_name = "TestPlainContent-%f" % time.time()
self.container_client.container_create(account=self.account,
reference=self.container_name)
self.container_id = cid_from_name(self.account,
self.container_name).upper()
self.content = "%s-%s" % (self.__class__.__name__, random_str(4))
self.stgpol = "SINGLE"
self.stgpol_twocopies = "TWOCOPIES"
self.stgpol_threecopies = "THREECOPIES"
def convert_to_old_chunk(chunk_path,
account,
container,
path,
version,
content_id,
add_old_fullpath=False):
chunk_id = chunk_path.rsplit('/', 1)[1]
cid = cid_from_name(account, container)
with open(chunk_path) as fd:
xattr.setxattr(fd, 'user.' + CHUNK_XATTR_KEYS['chunk_id'],
chunk_id.encode('utf-8'))
xattr.setxattr(fd, 'user.' + CHUNK_XATTR_KEYS['container_id'],
cid.encode('utf-8'))
xattr.setxattr(fd, 'user.' + CHUNK_XATTR_KEYS['content_path'],
path.encode('utf-8'))
xattr.setxattr(fd, 'user.' + CHUNK_XATTR_KEYS['content_version'],
str(version).encode('utf-8'))
xattr.setxattr(fd, 'user.' + CHUNK_XATTR_KEYS['content_id'],
content_id.encode('utf-8'))
if add_old_fullpath:
old_fullpath = encode_old_fullpath(account, container, path,
version).encode('utf-8')
hasher = sha256()
hasher.update(old_fullpath)
hash_old_fullpath = hasher.hexdigest().upper()
xattr.setxattr(fd, 'user.oio:' + hash_old_fullpath, old_fullpath)
xattr.setxattr(fd, 'user.' + CHUNK_XATTR_KEYS['oio_version'], b'4.0')
try:
xattr.removexattr(
fd, 'user.' + CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + chunk_id)
except IOError:
pass
def _get_object_metadata_cache_key(account=None, reference=None, path=None,
cid=None):
if not path:
raise ValueError('Missing object name to use the cache')
cid = cid or cid_from_name(account, reference)
cid = cid.upper()
return '/'.join(("meta", cid, path))
def _process_item(self, item):
namespace, account, container = item
if namespace != self.tool.namespace:
raise ValueError('Invalid namespace (actual=%s, expected=%s)' % (
namespace, self.tool.namespace))
errors = list()
if self.rebuild_bases:
cid = cid_from_name(account, container)
for res in self.meta2_database.rebuild(cid):
if res['err']:
errors.append('%s: %s' % (res['base'], res['err']))
if errors:
raise Exception(errors)
if self.sync_bases:
data = self.admin_client.election_sync(
service_type='meta2', account=account, reference=container)
for host, info in data.items():
if info['status']['status'] not in (200, 301):
errors.append('%s (%d): %s' % (
host, info['status']['status'],
info['status']['message']))
if errors:
raise Exception(errors)
if self.update_account:
self.container_client.container_touch(
account=account, reference=container)
def setUp(self):
super(TestBlobConverter, self).setUp()
self.container = random_str(16)
self.path = random_str(16)
self.api = ObjectStorageApi(self.ns)
self.api.container_create(self.account, self.container)
_, chunks = self.api.container.content_prepare(
self.account, self.container, self.path, size=1)
services = self.conscience.all_services('rawx')
self.rawx_volumes = dict()
for rawx in services:
tags = rawx['tags']
service_id = tags.get('tag.service_id', None)
if service_id is None:
service_id = rawx['addr']
volume = tags.get('tag.vol', None)
self.rawx_volumes[service_id] = volume
self.api.object_create(
self.account, self.container, obj_name=self.path, data="chunk")
meta, self.chunks = self.api.object_locate(
self.account, self.container, self.path)
self.version = meta['version']
self.content_id = meta['id']
self.container_id = cid_from_name(self.account, self.container)
def meta2_index_delete(self, volume_id, container_path=None,
container_id=None, **kwargs):
"""
Remove a meta2 record from the volume's index. Either the container ID
or the container path have to be given.
:param volume_id: The meta2 volume.
:param container_id: The container ID.
:param container_path: The container path
"""
if not container_path and container_id:
container_path = self._resolve_cid_to_path(container_id)
elif container_path and not container_id:
_tmp = container_path.rsplit("/")
container_id = cid_from_name(_tmp[1], _tmp[3])
elif not container_path and not container_id:
raise ValueError("At least the container ID or the container path "
"should be given.")
body = {'container_url': container_path,
'container_id': container_id}
for key, value in kwargs.iteritems():
body[key] = value
return self._rdir_request(volume=volume_id, method='POST',
action='delete', create=False, json=body,
service_type='meta2', **kwargs)
def setUp(self):
super(TestBlobMover, self).setUp()
self.container = random_str(16)
self.cid = cid_from_name(self.account, self.container)
self.path = random_str(16)
self.api = ObjectStorageApi(self.ns)
self.blob_client = BlobClient(self.conf)
self.api.container_create(self.account, self.container)
_, chunks = self.api.container.content_prepare(self.account,
self.container,
self.path,
size=1)
services = self.conscience.all_services('rawx')
if len(chunks) >= len([s for s in services if s['score'] > 0]):
self.skipTest("need at least %d rawx to run" % (len(chunks) + 1))
self.rawx_volumes = dict()
for rawx in services:
tags = rawx['tags']
service_id = tags.get('tag.service_id', None)
if service_id is None:
service_id = rawx['addr']
volume = tags.get('tag.vol', None)
self.rawx_volumes[service_id] = volume
self.api.object_create(self.account,
self.container,
obj_name=self.path,
data="chunk")
meta, self.chunks = self.api.object_locate(self.account,
self.container, self.path)
self.version = meta['version']
self.content_id = meta['id']
self.chunk_method = meta['chunk_method']
def _list_contents(self):
for container in self._list_containers():
marker = None
while True:
try:
_, listing = self.container_client.content_list(
account=self.account,
reference=container,
limit=self.content_fetch_limit,
marker=marker)
except NotFound:
self.logger.warn(
"Container %s appears in account but doesn't exist",
container)
break
if len(listing["objects"]) == 0:
break
for obj in listing["objects"]:
marker = obj["name"]
if obj["mtime"] > time.time() - self.outdated_threshold:
continue
if obj["policy"] == self.new_policy:
continue
container_id = cid_from_name(self.account, container)
yield (container_id, obj["content"])
def _put_chunk(self):
account = random_str(16)
container = random_str(16)
cid = cid_from_name(account, container)
content_path = random_str(16)
content_version = 1234567890
content_id = random_id(32)
fullpath = encode_fullpath(account, container, content_path,
content_version, content_id)
chunk_id = random_chunk_id()
data = random_buffer(string.printable, 100)
meta = {
'full_path': fullpath,
'container_id': cid,
'content_path': content_path,
'version': content_version,
'id': content_id,
'chunk_method': 'ec/algo=liberasurecode_rs_vand,k=6,m=3',
'policy': 'TESTPOLICY',
'chunk_hash': md5(data).hexdigest().upper(),
'oio_version': OIO_VERSION,
'chunk_pos': 0,
'metachunk_hash': md5().hexdigest(),
'metachunk_size': 1024
}
self.blob_client.chunk_put('http://' + self.rawx_id + '/' + chunk_id,
meta, data)
sleep(1) # ensure chunk event have been processed
return account, container, cid, content_path, content_version, \
content_id, chunk_id
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.test_dir = self.conf['sds_path']
self.chars = string.ascii_lowercase + string.ascii_uppercase +\
string.digits
self.chars_id = string.digits + 'ABCDEF'
self.rawx = 'http://' + self.conf["rawx"][0]['addr']
self.h = hashlib.new('md5')
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = rand_generator(self.chars, 8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = rand_generator(self.chars_id, 64)
self.data = rand_generator(self.chars, 1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(
rand_generator(self.chars, 6), len(self.data), self.url_rand, 1)
self.content.id_container = cid_from_name(
self.account, self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0,
self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0",
"size": self.chunk.size,
"url": self.chunk_url}
chunk_meta = {'content_size': self.content.size,
'content_chunksnb': self.content.nb_chunks,
'content_path': self.content.path,
'content_cid': self.content.id_container,
'content_mimetype': 'application/octet-stream',
'content_chunkmethod': 'bytes',
'content_policy': 'TESTPOLICY',
'content_id': '0000',
'content_version': 1,
'chunk_id': self.chunk.id_chunk,
'chunk_pos': self.chunk.pos}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = self.test_dir + '/data/NS-rawx-1/' +\
self.chunk.id_chunk[0:2] + "/" + self.chunk.id_chunk
self.bad_container_id = '0'*64
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.test_dir = self.conf['sds_path']
rawx_num, rawx_path, rawx_addr = self.get_service_url('rawx')
self.rawx = 'http://' + rawx_addr
self.h = hashlib.new('md5')
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = random_str(8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = random_id(64)
self.data = random_str(1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(random_str(6), len(self.data),
self.url_rand, 1)
self.content.id_container = cid_from_name(self.account,
self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0,
self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {
"hash": self.chunk.md5,
"pos": "0",
"size": self.chunk.size,
"url": self.chunk_url
}
chunk_meta = {
'content_path': self.content.path,
'container_id': self.content.id_container,
'chunk_method': 'plain/nb_copy=3',
'policy': 'TESTPOLICY',
'id': '0000',
'version': 1,
'chunk_id': self.chunk.id_chunk,
'chunk_pos': self.chunk.pos,
'chunk_hash': self.chunk.md5,
}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = self.test_dir + '/data/' + self.namespace + \
'-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk
self.bad_container_id = '0' * 64
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf["namespace"]
self.account = self.conf["account"]
self.test_dir = self.conf["sds_path"]
rawx_num, rawx_path, rawx_addr = self.get_service_url("rawx")
self.rawx = "http://" + rawx_addr
self.h = hashlib.new("md5")
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = random_str(8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = random_id(64)
self.data = random_str(1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(random_str(6), len(self.data), self.url_rand, 1)
self.content.id_container = cid_from_name(self.account, self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0, self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0", "size": self.chunk.size, "url": self.chunk_url}
chunk_meta = {
"content_path": self.content.path,
"container_id": self.content.id_container,
"chunk_method": "plain/nb_copy=3",
"policy": "TESTPOLICY",
"id": "0000",
"version": 1,
"chunk_id": self.chunk.id_chunk,
"chunk_pos": self.chunk.pos,
"chunk_hash": self.chunk.md5,
}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = (
self.test_dir
+ "/data/"
+ self.namespace
+ "-rawx-1/"
+ self.chunk.id_chunk[0:3]
+ "/"
+ self.chunk.id_chunk
)
self.bad_container_id = "0" * 64
def _containers_cleanup(self):
for container in self.containers:
self.container_client.container_delete(self.account, container)
for svc in self.containers_svcs[container]:
self.rdir_client.meta2_index_delete(
volume_id=svc['host'],
container_path="{0}/{1}/{2}".format(
self.ns, self.account, container),
container_id=cid_from_name(self.account, container))
def setUp(self):
super(TestMeta2EventsEmission, self).setUp()
self.container_name = 'TestEventsEmission%f' % time.time()
self.container_id = cid_from_name(self.account, self.container_name)
self.container_client = ContainerClient(self.conf)
self.storage_api = ObjectStorageApi(self.conf['namespace'])
self.event_agent_name = 'event-agent-1'
self.bt_connections = []
self._bt_make_connections(self.conf['services']['beanstalkd'])
def _setup(self):
self.container = 'blob'
self.cid = cid_from_name(self.account, 'blob')
self.content_path = 'test-plop'
self.content_version = '1456938361143740'
self.content_id = '0123456789ABCDEF'
self.fullpath = encode_fullpath(
self.account, 'blob', self.content_path, self.content_version,
self.content_id)
def apply(self, account, container, obj_meta, **kwargs):
if self.match(obj_meta, **kwargs):
cid = cid_from_name(account, container)
if not self.factory:
return "Kept"
# TODO: avoid loading content description a second time
self.factory.change_policy(cid, obj_meta['id'], self.policy)
return "Policy changed to %s" % self.policy
return "Kept"
def _verify_rebuilder_input(self):
try:
line = fileinput.input(self.rebuild_file).next().strip()
cid = cid_from_name(self.account, self.container)
expected = '|'.join([cid, self.meta['id'], self.chunk['url']])
if self.irreparable:
expected = IRREPARABLE_PREFIX + '|' + expected
self.assertEqual(expected, line)
finally:
fileinput.close()
def _fill_queue(self, queue, **kwargs):
if self._fill_queue_from_file(queue, **kwargs):
return
accounts = self.api.account_list()
for account in accounts:
containers = self._full_container_list(account)
for container in containers:
cid = cid_from_name(account, container[0])
queue.put(cid)
def test_container_locate(self):
container = random_str(10)
account = random_str(10)
opts = self.get_format_opts()
output = self.openio('container create --oio-account %s %s %s'
% (account, container, opts))
self.assertOutput('%s True' % container, output.strip())
output = self.openio_admin('container locate --oio-account %s %s %s' %
(account, container, opts))
cid = cid_from_name(account, container)
self._test_container_check_output(output, cid, account, container)
def format_m2(self, acct, ct, m2_srv):
cid = cid_from_name(acct, ct)
m2_item = self.m2_item(acct, ct, cid)
reqid = self.app.request_id(self.reqid_prefix)
all_m2 = self.all_services('meta2', reqid=reqid)
for m2 in m2_srv:
m2_descr = all_m2.get(m2['host'], DUMMY_SERVICE)
status = 'up=%s, score=%s' % (m2_descr['tags'].get(
'tag.up', False), m2_descr['score'])
yield ('meta2', m2_item, m2['host'], m2_descr['addr'],
m2_descr['tags'].get('tag.loc', 'n/a'), status, None)
def _safe_chunk_rebuild(self, item, content_id, chunk_id_or_pos, **kwargs):
_, account, container, _, _ = item
try:
container_id = cid_from_name(account, container)
self.chunk_operator.rebuild(container_id, content_id,
chunk_id_or_pos, **kwargs)
except Exception as exc: # pylint: disable=broad-except
self.logger.error('ERROR when rebuilding chunk %s (%s): %s',
self.tool.string_from_item(item),
chunk_id_or_pos, exc)
return exc
def _convert_and_check(self,
chunk_volume,
chunk_path,
chunk_id_info,
expected_raw_meta=None,
expected_errors=0):
conf = self.conf
conf['volume'] = self.rawx_volumes[chunk_volume]
converter = BlobConverter(conf, logger=self.logger)
converter.safe_convert_chunk(chunk_path)
self.assertEqual(1, converter.total_chunks_processed)
self.assertEqual(1, converter.passes)
self.assertEqual(expected_errors, converter.errors)
checker = Checker(self.ns)
for chunk_id, info in chunk_id_info.items():
account, container, path, version, content_id = info
fullpath = encode_fullpath(account, container, path, version,
content_id)
cid = cid_from_name(account, container)
meta, raw_meta = read_chunk_metadata(chunk_path, chunk_id)
self.assertEqual(meta.get('chunk_id'), chunk_id)
self.assertEqual(meta.get('container_id'), cid)
self.assertEqual(meta.get('content_path'), path)
self.assertEqual(meta.get('content_version'), version)
self.assertEqual(meta.get('content_id'), content_id)
self.assertEqual(meta.get('full_path'), fullpath)
checker.check(
Target(account,
container=container,
obj=path,
chunk='http://' + converter.volume_id + '/' + chunk_id))
for _ in checker.run():
pass
self.assertTrue(checker.report())
if expected_raw_meta:
self.assertDictEqual(expected_raw_meta, raw_meta)
continue
self.assertNotIn(CHUNK_XATTR_KEYS['chunk_id'], raw_meta)
self.assertNotIn(CHUNK_XATTR_KEYS['container_id'], raw_meta)
self.assertNotIn(CHUNK_XATTR_KEYS['content_path'], raw_meta)
self.assertNotIn(CHUNK_XATTR_KEYS['content_version'], raw_meta)
self.assertNotIn(CHUNK_XATTR_KEYS['content_id'], raw_meta)
self.assertIn(CHUNK_XATTR_CONTENT_FULLPATH_PREFIX + chunk_id,
raw_meta)
for k in raw_meta.keys():
if k.startswith('oio:'):
self.fail('old fullpath always existing')
self.assertEqual(raw_meta[CHUNK_XATTR_KEYS['oio_version']],
OIO_VERSION)
def __init__(self, account, ref):
self.cid = cid_from_name(account, ref)
self.path = random_str(6)
self.version = 1
self.id = random_id(32)
self.fullpath = encode_fullpath(account, ref, self.path, self.version,
self.id)
self.data = os.urandom(1280)
self.size = len(self.data)
md5 = hashlib.new('md5')
md5.update(self.data)
self.hash = md5.hexdigest().lower()
def setUp(self):
super(TestContentFactory, self).setUp()
self.namespace = self.conf['namespace']
self.chunk_size = self.conf['chunk_size']
self.gridconf = {"namespace": self.namespace}
self.content_factory = ContentFactory(self.gridconf)
self.container_name = "TestContentFactory%f" % time.time()
self.container_client = ContainerClient(self.gridconf)
self.container_client.container_create(acct=self.account,
ref=self.container_name)
self.container_id = cid_from_name(self.account,
self.container_name).upper()
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.test_dir = self.conf['sds_path']
rawx_num, rawx_path, rawx_addr = self.get_service_url('rawx')
self.rawx = 'http://' + rawx_addr
self.h = hashlib.new('md5')
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = random_str(8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = random_id(64)
self.data = random_str(1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(
random_str(6), len(self.data), self.url_rand, 1)
self.content.id_container = cid_from_name(
self.account, self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0,
self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0",
"size": self.chunk.size,
"url": self.chunk_url}
chunk_meta = {'content_path': self.content.path,
'container_id': self.content.id_container,
'content_chunkmethod': 'plain/nb_copy=3',
'content_policy': 'TESTPOLICY',
'content_id': '0000',
'content_version': 1,
'chunk_id': self.chunk.id_chunk,
'chunk_pos': self.chunk.pos}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = self.test_dir + '/data/' + self.namespace + \
'-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk
self.bad_container_id = '0'*64
def _run(self, parsed_args):
meta2 = Meta2Database(self.app.client_manager.client_conf,
logger=self.log)
for container in parsed_args.containers:
if parsed_args.is_cid:
cid = container
else:
cid = cid_from_name(self.app.options.account, container)
moved = meta2.move(cid, parsed_args.src, dst=parsed_args.dst)
for res in moved:
res['container'] = container
yield res
def test_show_by_cid(self):
name = random_str(32)
self._create(name)
res = self.api.list(cid=cid_from_name(self.account, name))
self.assertIsNotNone(res['dir'])
self.assertIsNotNone(res['srv'])
self.assertEqual(res['name'], name)
self.assertEqual(res['account'], self.account)
self._delete(name)
def setUp(self):
super(TestContentFactory, self).setUp()
self.namespace = self.conf['namespace']
self.chunk_size = self.conf['chunk_size']
self.gridconf = {"namespace": self.namespace}
self.content_factory = ContentFactory(self.gridconf)
self.container_name = "TestContentFactory%f" % time.time()
self.container_client = ContainerClient(self.gridconf)
self.container_client.container_create(acct=self.account,
ref=self.container_name)
self.container_id = cid_from_name(self.account,
self.container_name).upper()
def __init__(self, test_conf, account,
container_name, content_name, stgpol):
self.test_conf = test_conf
self.account = account
self.container_name = container_name
self.content_name = content_name
self.content_id = generate_id(32)
self.container_id = cid_from_name(self.account,
self.container_name).upper()
self.hash = 32 * '0'
self.version = "0"
self.size = 0
self.chunks = []
self.stgpol = stgpol
def setUp(self):
with mock.patch('oio.container.client.gen_headers',
gen_headers_mock):
super(TestFilters, self).setUp()
self.account = self.conf['account']
self.namespace = self.conf['namespace']
self.chunk_size = self.conf['chunk_size']
self.gridconf = {'namespace': self.namespace}
self.content_factory = ContentFactory(self.gridconf)
self.container_name = 'TestFilter%f' % time.time()
self.blob_client = BlobClient()
self.container_client = ContainerClient(self.gridconf)
self.container_client.container_create(acct=self.account,
ref=self.container_name)
self.container_id = cid_from_name(self.account,
self.container_name).upper()
self.stgpol = "SINGLE"
def setUp(self):
super(TestContentFactory, self).setUp()
self.namespace = self.conf['namespace']
self.chunk_size = self.conf['chunk_size']
self.gridconf = {"namespace": self.namespace}
self.content_factory = ContentFactory(self.gridconf)
self.container_name = "TestContentFactory%f" % time.time()
self.blob_client = BlobClient()
self.container_client = ContainerClient(self.gridconf)
self.container_client.container_create(acct=self.account,
ref=self.container_name)
self.container_id = cid_from_name(self.account,
self.container_name).upper()
self.stgpol = "SINGLE"
self.stgpol_twocopies = "TWOCOPIES"
self.stgpol_threecopies = "THREECOPIES"
self.stgpol_ec = "EC"
def setUp(self):
super(TestDupContent, self).setUp()
if len(self.conf['rawx']) < 3:
self.skipTest("Not enough rawx. "
"Dup tests needs more than 2 rawx to run")
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.chunk_size = self.conf['chunk_size']
self.gridconf = {"namespace": self.namespace}
self.content_factory = ContentFactory(self.gridconf)
self.container_client = ContainerClient(self.gridconf)
self.blob_client = BlobClient()
self.container_name = "TestDupContent%f" % time.time()
self.container_client.container_create(acct=self.account,
ref=self.container_name)
self.container_id = cid_from_name(self.account,
self.container_name).upper()
def _list_contents(self):
for container in self._list_containers():
marker = None
while True:
try:
resp = self.container_client.container_list(
acct=self.account, ref=container,
limit=self.content_fetch_limit, marker=marker)
except NotFound:
self.logger.warn("Container %s in account "
"but not found" % container)
break
if len(resp["objects"]) == 0:
break
for obj in resp["objects"]:
marker = obj["name"]
if obj["mtime"] > time.time() - self.outdated_threshold:
continue
if obj["policy"] == self.new_policy:
continue
container_id = cid_from_name(self.account, container)
yield (container_id, obj["content"])
def setUp(self):
super(TestPlainContent, self).setUp()
if len(self.conf['services']['rawx']) < 4:
self.skipTest(
"Plain tests needs more than 3 rawx to run")
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.chunk_size = self.conf['chunk_size']
self.gridconf = {"namespace": self.namespace}
self.content_factory = ContentFactory(self.gridconf)
self.container_client = ContainerClient(self.gridconf)
self.blob_client = BlobClient()
self.container_name = "TestPlainContent-%f" % time.time()
self.container_client.container_create(acct=self.account,
ref=self.container_name)
self.container_id = cid_from_name(self.account,
self.container_name).upper()
self.content = random_str(64)
self.stgpol = "SINGLE"
self.stgpol_twocopies = "TWOCOPIES"
self.stgpol_threecopies = "THREECOPIES"
def setUp(self):
super(TestECContent, self).setUp()
if len(self.conf['services']['rawx']) < 12:
self.skipTest("Not enough rawx. "
"EC tests needs at least 12 rawx to run")
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.chunk_size = self.conf['chunk_size']
self.gridconf = {"namespace": self.namespace}
self.content_factory = ContentFactory(self.gridconf)
self.container_client = ContainerClient(self.gridconf)
self.blob_client = BlobClient()
self.container_name = "TestECContent%f" % time.time()
self.container_client.container_create(acct=self.account,
ref=self.container_name)
self.container_id = cid_from_name(self.account,
self.container_name).upper()
self.content = random_str(64)
self.stgpol = "EC"
self.size = 1024*1024 + 320
self.k = 6
self.m = 3
