def _record(self):
return {
"container_id": random_id(64),
"content_id": random_id(32),
"chunk_id": random_id(64),
"mtime": int(time.time())
}
def test_chunk_fetch_with_rebuild_container_id_start_after_limit(self):
self.rdir.admin_incident_set(self.rawx_id, self.incident_date)
self.rdir._direct_request.reset_mock()
expected_entries_rebuild = [
entry for entry in self.expected_entries if entry[2][-1] == '0'
]
cid = random.choice(self.expected_entries)[0]
expected_entries_rebuild_cid = \
[entry for entry in expected_entries_rebuild
if entry[0] == cid]
if expected_entries_rebuild_cid:
start_after_index = random.randrange(
0, len(expected_entries_rebuild_cid))
start_after = '|'.join(
expected_entries_rebuild_cid[start_after_index][:3])
else:
start_after_index = 0
start_after = '|'.join(
(random_id(64), random_id(32), random_id(64)))
entries = self.rdir.chunk_fetch(self.rawx_id,
rebuild=True,
container_id=cid,
start_after=start_after,
limit=2)
self._assert_chunk_fetch(
expected_entries_rebuild_cid[start_after_index + 1:],
entries,
limit=2)
def setUp(self) -> None:
# Given
self.test_type = random_id()
self.test_id = random_id()
self.test_uuid = random_uuid()
self.test_case = make_ingest_entity(self.test_type, self.test_id, self.test_uuid)
self.submission = HcaSubmission()
# When
self.test_entity = self.submission.map_ingest_entity(self.test_case)
def _create_chunk(self, rawx_path, alias="toto", suffix=''):
cname = random_str(8)
container_id = cid_from_name(self.account, cname)
content_id = random_id(32)
chunk_id = random_id(64)
chunk_dir = "%s/%s" % (rawx_path, chunk_id[0:3])
if not os.path.isdir(chunk_dir):
os.makedirs(chunk_dir)
chunk_path = "%s/%s%s" % (chunk_dir, chunk_id, suffix)
with open(chunk_path, "w") as chunk_file:
chunk_file.write("toto")
# pylint: disable=no-member
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['chunk_hash'], 32 * b'0')
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['chunk_id'], enc(chunk_id))
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['chunk_pos'], b'0')
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['chunk_size'], b'4')
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['content_policy'],
b'TESTPOLICY')
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['content_chunkmethod'],
b'plain/nb_copy=3')
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['content_version'], b'1')
# Old (oio-sds < 4.2) extended attributes
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['container_id'],
enc(container_id))
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['content_id'],
enc(content_id))
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['content_path'], enc(alias))
# New (oio-sds >= 4.2) extended attributes
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['oio_version'],
enc(OIO_VERSION))
fullpath = encode_fullpath(self.account, cname, alias, 1, content_id)
xattr.setxattr(
chunk_path,
'user.%s%s' % (CHUNK_XATTR_CONTENT_FULLPATH_PREFIX, chunk_id),
enc(fullpath))
return chunk_path, container_id, content_id, chunk_id
def setUp(self):
self.endpoint = "http://1.2.3.4:8000"
self.api = FakeDirectoryAPI("NS", self.endpoint)
self.account = "AUTH_test"
self.headers = {"x-req-id": random_id(64)}
self.uri_base = "NS"
self.name = random_str(64)
def __init__(self, rawx_id, rawx_loc, metachunk_size, metachunk_hash):
self.id = random_id(64)
self.url = "%s/%s" % (rawx_id, self.id)
self.path = rawx_loc + '/' + self.id[0:3] + '/' + self.id
self.pos = 0
self.metachunk_size = metachunk_size
self.metachunk_hash = metachunk_hash
def test_rdir_clear_and_lock(self):
rec = self._record()
who = random_id(32)
# push with autocreate
resp = self._post(
"/v1/rdir/push", params={'vol': self.vol, 'create': True},
data=json.dumps(rec))
self.assertEqual(resp.status_code, 204)
# lock
resp = self._post(
"/v1/rdir/admin/lock", params={'vol': self.vol},
data=json.dumps({'who': who}))
self.assertEqual(resp.status_code, 204)
# try to clear while the lock is held
resp = self._post("/v1/rdir/admin/clear", params={'vol': self.vol})
self.assertEqual(resp.status_code, 403)
# unlock
resp = self._post("/v1/rdir/admin/unlock", params={'vol': self.vol})
self.assertEqual(resp.status_code, 204)
# clear all entries
resp = self._post(
"/v1/rdir/admin/clear", params={'vol': self.vol},
data=json.dumps({'all': True}))
self.assertEqual(resp.status_code, 200)
self.assertEqual(resp.json(), {'removed': 1})
def test_rdir_clear_with_repair(self):
rec = self._record()
who = random_id(32)
# clear without volume
resp = self._post("/v1/rdir/admin/clear", params={'repair': True})
self.assertEqual(resp.status, 400)
# push with autocreate
resp = self._post(
"/v1/rdir/push", params={'vol': self.vol, 'create': True},
data=json.dumps(rec))
self.assertEqual(resp.status, 204)
# lock
resp = self._post(
"/v1/rdir/admin/lock", params={'vol': self.vol},
data=json.dumps({'who': who}))
self.assertEqual(resp.status, 204)
# try to clear while the lock is held
resp = self._post("/v1/rdir/admin/clear",
params={'vol': self.vol, 'repair': True})
self.assertEqual(resp.status, 403)
# unlock
resp = self._post("/v1/rdir/admin/unlock", params={'vol': self.vol})
self.assertEqual(resp.status, 204)
# repair all entries
resp = self._post("/v1/rdir/admin/clear",
params={'vol': self.vol, 'repair': True})
self.assertEqual(resp.status, 200)
self.assertDictEqual(self.json_loads(resp.data),
{'removed': 0, 'repaired': 1, 'errors': 0})
def test_send_all_samples_sends_each_samples_with_project_release_date(self):
# Given
self.submitter.send_sample = MagicMock(return_value='UPDATED')
test_date = datetime.now().isoformat()
project_attributes = {
'releaseDate': test_date
}
project_attributes = make_ingest_entity('projects', random_id(), random_uuid(), project_attributes)
self.submission.map_ingest_entity(project_attributes)
biomaterials = self.map_random_biomaterials(self.submission, 5)
calls = []
call_count = 0
expected_result = {'UPDATED': []}
for entity in biomaterials:
call_count += 1
calls.append(call(entity, test_date))
expected_result['UPDATED'].append(entity)
# When
result = self.submitter.send_all_samples(self.submission)
# Then
self.submitter.send_sample.assert_has_calls(calls, any_order=True)
self.assertEqual(call_count, self.submitter.send_sample.call_count)
self.assertDictEqual(expected_result, result)
def test_bad_routes(self):
routes = ('/status', '/config',
'/v1/status',
'/v1/rdir/admin/show',
'/v1/rdir/admin/lock',
'/v1/rdir/admin/unlock',
'/v1/rdir/admin/incident',
'/v1/rdir/admin/clear',
'/v1/rdir/create',
'/v1/rdir/push',
'/v1/rdir/delete',
'/v1/rdir/fetch',
'/v1/rdir/status',
'/v1/rdir/meta2/create',
'/v1/rdir/meta2/push',
'/v1/rdir/meta2/delete',
'/v1/rdir/meta2/fetch')
for r in routes:
resp = self._get('/' + r)
self.assertEqual(resp.status, 404)
resp = self._get(r + '/')
self.assertEqual(resp.status, 404)
bulk = random_id(4)
resp = self._get('/' + bulk + r)
self.assertEqual(resp.status, 404)
resp = self._get(r + '/' + bulk)
self.assertEqual(resp.status, 404)
resp = self._get(r + bulk)
self.assertEqual(resp.status, 404)
def setUp(self):
self.endpoint = "http://1.2.3.4:8000"
self.api = FakeDirectoryAPI("NS", self.endpoint)
self.account = "AUTH_test"
self.headers = {"x-req-id": random_id(64)}
self.uri_base = "NS"
self.name = random_str(64)
def test_copy_with_same_chunkid(self):
metachunk_hash = md5().hexdigest()
trailers = {
'x-oio-chunk-meta-metachunk-size': 1,
'x-oio-chunk-meta-metachunk-hash': metachunk_hash
}
chunkid1 = random_chunk_id()
chunkdata1 = random_buffer(string.printable, 1)
chunkurl1 = self._rawx_url(chunkid1)
headers1 = self._chunk_attr(chunkid1, chunkdata1)
self._check_not_present(chunkurl1)
resp, _ = self._http_request(chunkurl1, 'PUT', chunkdata1, headers1,
trailers)
self.assertEqual(201, resp.status)
self.assertEqual(headers1['x-oio-chunk-meta-chunk-hash'].upper(),
resp.getheader('x-oio-chunk-meta-chunk-hash'))
self.assertEqual(headers1['x-oio-chunk-meta-chunk-size'].upper(),
resp.getheader('x-oio-chunk-meta-chunk-size'))
headers = {}
headers["Destination"] = chunkurl1
headers['x-oio-chunk-meta-full-path'] = encode_fullpath(
"account-snapshot", "container-snapshot", "content-snapshot",
1456938361143741, random_id(32))
resp, _ = self._http_request(chunkurl1, 'COPY', '', headers)
self.assertEqual(403, resp.status)
def test_admin_show(self):
who = random_id(32)
self.rdir.admin_lock(self.volume, who)
self.rdir.admin_set_incident_date(self.volume, 1234)
res = self.rdir.admin_show(self.volume)
self.assertEqual(res, {'incident_date': "1234",
'lock': who})
def _put_chunk(self):
account = random_str(16)
container = random_str(16)
cid = cid_from_name(account, container)
content_path = random_str(16)
content_version = 1234567890
content_id = random_id(32)
fullpath = encode_fullpath(account, container, content_path,
content_version, content_id)
chunk_id = random_chunk_id()
data = random_buffer(string.printable, 100)
meta = {
'full_path': fullpath,
'container_id': cid,
'content_path': content_path,
'version': content_version,
'id': content_id,
'chunk_method': 'ec/algo=liberasurecode_rs_vand,k=6,m=3',
'policy': 'TESTPOLICY',
'chunk_hash': md5(data).hexdigest().upper(),
'oio_version': OIO_VERSION,
'chunk_pos': 0,
'metachunk_hash': md5().hexdigest(),
'metachunk_size': 1024
}
self.blob_client.chunk_put('http://' + self.rawx_id + '/' + chunk_id,
meta, data)
sleep(1) # ensure chunk event have been processed
return account, container, cid, content_path, content_version, \
content_id, chunk_id
def test_copy_with_nonexistent_source(self):
metachunk_hash = md5().hexdigest()
trailers = {'x-oio-chunk-meta-metachunk-size': '1',
'x-oio-chunk-meta-metachunk-hash': metachunk_hash}
chunkid1 = random_chunk_id()
chunkurl1 = self._rawx_url(chunkid1)
chunkid2 = random_chunk_id()
chunkdata2 = random_buffer(string.printable, 1).encode('utf-8')
chunkurl2 = self._rawx_url(chunkid2)
headers2 = self._chunk_attr(chunkid2, chunkdata2)
self._check_not_present(chunkurl2)
resp, _ = self._http_request(chunkurl2, 'PUT', chunkdata2, headers2,
trailers)
self.assertEqual(201, resp.status)
self.assertEqual(headers2['x-oio-chunk-meta-chunk-hash'].upper(),
resp.getheader('x-oio-chunk-meta-chunk-hash'))
self.assertEqual(headers2['x-oio-chunk-meta-chunk-size'],
resp.getheader('x-oio-chunk-meta-chunk-size'))
headers = {}
headers["Destination"] = chunkurl2
headers['x-oio-chunk-meta-full-path'] = encode_fullpath(
"account-snapshot", "container-snapshot", "content-snapshot",
1456938361143741, random_id(32))
resp, _ = self._http_request(chunkurl1, 'COPY', '', headers)
self.assertEqual(404, resp.status)
def _create_content(self, name):
headers = {'X-oio-action-mode': 'autocreate'}
params = self.param_content(self.ref, name)
resp = self.request('POST',
self.url_content('prepare'),
params=params,
headers=headers,
data=json.dumps({'size': '1024'}))
self.assertEqual(resp.status, 200)
chunks = self.json_loads(resp.data)
stgpol = resp.getheader('x-oio-content-meta-policy')
headers = {
'x-oio-action-mode': 'autocreate',
'x-oio-content-meta-size': '1024',
'x-oio-content-meta-policy': stgpol,
'x-oio-content-meta-version': int(time.time() * 1000000),
'x-oio-content-meta-id': random_id(32)
}
resp = self.request('POST',
self.url_content('create'),
params=params,
headers=headers,
data=json.dumps(chunks))
self.assertEqual(resp.status, 204)
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.test_dir = self.conf['sds_path']
rawx_num, rawx_path, rawx_addr = self.get_service_url('rawx')
self.rawx = 'http://' + rawx_addr
self.h = hashlib.new('md5')
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = random_str(8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = random_id(64)
self.data = random_str(1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(random_str(6), len(self.data),
self.url_rand, 1)
self.content.id_container = cid_from_name(self.account,
self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0,
self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {
"hash": self.chunk.md5,
"pos": "0",
"size": self.chunk.size,
"url": self.chunk_url
}
chunk_meta = {
'content_path': self.content.path,
'container_id': self.content.id_container,
'chunk_method': 'plain/nb_copy=3',
'policy': 'TESTPOLICY',
'id': '0000',
'version': 1,
'chunk_id': self.chunk.id_chunk,
'chunk_pos': self.chunk.pos,
'chunk_hash': self.chunk.md5,
}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = self.test_dir + '/data/' + self.namespace + \
'-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk
self.bad_container_id = '0' * 64
def map_random_entities(
submission: HcaSubmission,
entity_count: int,
entity_type: str = None
):
entity_list = []
for i in range(0, entity_count):
entity_list.append(
submission.map_ingest_entity(
make_ingest_entity(
entity_type if entity_type else random_id(),
random_id(),
random_uuid()
)
)
)
return entity_list
def setUp(self):
self.endpoint = "http://1.2.3.4:8000"
self.api = FakeDirectoryClient({'namespace': "NS"},
endpoint=self.endpoint)
self.account = "AUTH_test"
self.headers = {"x-req-id": random_id(64)}
self.uri_base = "%s/v3.0/NS" % self.endpoint
self.name = random_str(64)
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf["namespace"]
self.account = self.conf["account"]
self.test_dir = self.conf["sds_path"]
rawx_num, rawx_path, rawx_addr = self.get_service_url("rawx")
self.rawx = "http://" + rawx_addr
self.h = hashlib.new("md5")
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = random_str(8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = random_id(64)
self.data = random_str(1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(random_str(6), len(self.data), self.url_rand, 1)
self.content.id_container = cid_from_name(self.account, self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0, self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0", "size": self.chunk.size, "url": self.chunk_url}
chunk_meta = {
"content_path": self.content.path,
"container_id": self.content.id_container,
"chunk_method": "plain/nb_copy=3",
"policy": "TESTPOLICY",
"id": "0000",
"version": 1,
"chunk_id": self.chunk.id_chunk,
"chunk_pos": self.chunk.pos,
"chunk_hash": self.chunk.md5,
}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = (
self.test_dir
+ "/data/"
+ self.namespace
+ "-rawx-1/"
+ self.chunk.id_chunk[0:3]
+ "/"
+ self.chunk.id_chunk
)
self.bad_container_id = "0" * 64
def setUp(self):
super(TestRdirClient, self).setUp()
self.namespace = self.conf["namespace"]
self.rdir_client = RdirClient({"namespace": self.namespace})
self.rdir_client._get_rdir_addr = Mock(return_value="0.1.2.3:4567")
self.container_id_1 = random_id(64)
self.container_id_2 = random_id(64)
self.container_id_3 = random_id(64)
self.content_id_1 = random_id(32)
self.content_id_2 = random_id(32)
self.content_id_3 = random_id(32)
self.chunk_id_1 = random_id(64)
self.chunk_id_2 = random_id(64)
self.chunk_id_3 = random_id(64)
def setUp(self):
super(TestRdirClient, self).setUp()
self.namespace = self.conf['namespace']
self.rdir_client = RdirClient({'namespace': self.namespace})
self.rdir_client._get_rdir_addr = Mock(return_value="0.1.2.3:4567")
self.container_id_1 = random_id(64)
self.container_id_2 = random_id(64)
self.container_id_3 = random_id(64)
self.content_id_1 = random_id(32)
self.content_id_2 = random_id(32)
self.content_id_3 = random_id(32)
self.chunk_id_1 = random_id(64)
self.chunk_id_2 = random_id(64)
self.chunk_id_3 = random_id(64)
def map_random_biomaterials(submission: HcaSubmission, entity_count: int) -> List[Entity]:
biomaterials = []
for i in range(0, entity_count):
biomaterials.append(
submission.map_ingest_entity(
make_ingest_entity('biomaterials', random_id(), random_uuid())
)
)
return biomaterials
def test_attributes_set_correctly_when_biosamples_accession_is_added_to_attributes(self):
# When
random_accession = random_id()
self.test_entity.add_accession('BioSamples', random_accession)
self.submission.add_accessions_to_attributes(self.test_entity)
# Then
acc = self.test_entity.attributes['content']['biomaterial_core']['biosamples_accession']
self.assertEqual(random_accession, acc)
def setUp(self):
super(TestRdirClient, self).setUp()
self.namespace = "dummy"
self.rdir_client = RdirClient({'namespace': self.namespace},
endpoint='127.0.0.0:6000')
self.rdir_client._get_rdir_addr = Mock(return_value="0.1.2.3:4567")
self.container_id_1 = random_id(64)
self.container_id_2 = random_id(64)
self.container_id_3 = random_id(64)
self.content_id_1 = random_id(32)
self.content_id_2 = random_id(32)
self.content_id_3 = random_id(32)
self.chunk_id_1 = random_id(64)
self.chunk_id_2 = random_id(64)
self.chunk_id_3 = random_id(64)
def test_admin_lock_unlock(self):
# unlock without lock
self.rdir.admin_unlock(self.volume)
# lock
who = random_id(32)
res = self.rdir.admin_lock(self.volume, who)
self.assertEqual(res, None)
# double lock
res = self.rdir.admin_lock(self.volume, random_id(32))
self.assertEqual(res, who)
# unlock
self.rdir.admin_unlock(self.volume)
# lock again
res = self.rdir.admin_lock(self.volume, random_id(32))
self.assertEqual(res, None)
def _meta2_record(self):
return {
"container_id":
random_id(64),
"container_url":
"{0}/{1}/{2}".format(random_str(12), random_str(12),
random_str(12)),
"mtime":
int(time.time())
}
def test_cycle_content(self):
path = random_content()
headers = {'x-oio-action-mode': 'autocreate'}
params = self.param_content(self.ref, path)
resp = self.request('GET', self.url_content('show'), params=params)
self.assertError(resp, 404, 406)
resp = self.request('POST', self.url_content('touch'), params=params)
self.assertError(resp, 404, 406)
resp = self.request('POST',
self.url_content('prepare'),
data=json.dumps({'size': '1024'}),
params=params,
headers=headers)
self.assertEqual(resp.status, 200)
chunks = self.json_loads(resp.data)
stgpol = resp.getheader('x-oio-content-meta-policy')
headers = {
'x-oio-action-mode': 'autocreate',
'x-oio-content-meta-size': '1024',
'x-oio-content-meta-policy': stgpol,
'x-oio-content-meta-version': int(time.time() * 1000000),
'x-oio-content-meta-id': random_id(32)
}
resp = self.request('POST',
self.url_content('create'),
params=params,
headers=headers,
data=json.dumps(chunks))
self.assertEqual(resp.status, 204)
# # FIXME check re-create depending on the container's ver'policy
# resp = self.request('POST', self.url_content('create'),
# params=params,
# headers=headers,
# data=json.dumps(chunks))
# self.assertEqual(resp.status, 201)
resp = self.request('GET', self.url_content('show'), params=params)
self.assertEqual(resp.status, 200)
resp = self.request('GET', self.url_content('show'), params=params)
self.assertEqual(resp.status, 200)
resp = self.request('POST', self.url_content('delete'), params=params)
self.assertEqual(resp.status, 204)
resp = self.request('GET', self.url_content('show'), params=params)
self.assertError(resp, 404, 420)
resp = self.request('POST', self.url_content('delete'), params=params)
self.assertError(resp, 404, 420)
def test_submit_sample_return_error(self):
# Given
error_msg = f'TestCaseError{random_id(3)}'
self.biosamples.send_sample = MagicMock(side_effect=Exception(error_msg))
test_case = self.submission.map_ingest_entity(
make_ingest_entity('biomaterials', random_id(), random_uuid()))
# When
result = self.submitter.send_sample(test_case)
# Then
self.assertEqual('ERRORED', result)
self.assertDictEqual(test_case.get_errors(), {'content.biomaterial_core.biosamples_accession': [f'BioSamples Error: {error_msg}']})
def test_worm_and_admin(self):
if not os.getenv("WORM"):
self.skipTest("must be in worm mode")
data = random_data(10)
path = 'test_worm'
content = self._new_content(data, path, True)
# Prepare without admin mode
self.assertRaises(Conflict, self._prepare_content, path, None, False)
self.assertRaises(Conflict, self._prepare_content, path,
content.content_id, False)
self.assertRaises(Conflict, self._prepare_content, 'test_worm_prepare',
content.content_id, False)
self.assertRaises(Conflict, self._prepare_content, path, random_id(32),
False)
# Overwrite without admin mode
data2 = random_data(11)
self.assertRaises(Conflict, self._new_content, data2, path, False)
# Prepare with admin mode
self._prepare_content(path, None, True)
self._prepare_content(path, content.content_id, True)
self._prepare_content('test_worm_prepare', content.content_id, True)
self._prepare_content(path, random_id(32), True)
# Overwrite with admin mode
content = self._new_content(data2, path, True)
# Delete without admin mode
try:
content.delete()
except ClientException as exc:
self.assertIn('worm', str(exc))
else:
self.fail("Delete without admin mode: no exception")
downloaded_data = ''.join(content.fetch())
self.assertEqual(downloaded_data, data2)
# Delete with admin mode
content.delete(admin_mode=True)
def test_worm_and_admin(self):
if not os.getenv("WORM"):
self.skipTest("must be in worm mode")
data = random_data(10)
path = 'test_worm'
content = self._new_content(data, path, True)
# Prepare without admin mode:
# Since the 'prepare' step is done in the proxy, there is no check
# on the pre-existence of the content. The subsequent prepare MUST
# now work despite the presence of the content.
self._prepare_content(path, None, False)
self._prepare_content(path, content.content_id, False)
self._prepare_content('test_worm_prepare', content.content_id, False)
self._prepare_content(path, random_id(32), False)
# Overwrite without admin mode
data2 = random_data(11)
self.assertRaises(Conflict, self._new_content, data2, path, False)
# Prepare with admin mode
self._prepare_content(path, None, True)
self._prepare_content(path, content.content_id, True)
self._prepare_content('test_worm_prepare', content.content_id, True)
self._prepare_content(path, random_id(32), True)
# Overwrite with admin mode
content = self._new_content(data2, path, True)
# Delete without admin mode
try:
content.delete()
except ClientException as exc:
self.assertIn('worm', str(exc))
else:
self.fail("Delete without admin mode: no exception")
downloaded_data = ''.join(content.fetch())
self.assertEqual(downloaded_data, data2)
# Delete with admin mode
content.delete(admin_mode=True)
def setUp(self):
super(TestRdirServer, self).setUp()
self.db_path = tempfile.mkdtemp()
self.conf = {'db_path': self.db_path,
'namespace': 'OPENIO'}
self.app = Client(create_app(self.conf), BaseResponse)
self.volume = 'testvolume'
self.app.get("/v1/rdir/create", query_string={'vol': self.volume})
self.container_id = random_id(64)
self.content_id = random_id(32)
self.chunk_id = random_id(64)
self.mtime = int(time.time())
self.rtime = 0
self.meta = {
'container_id': self.container_id,
'content_id': self.content_id,
'chunk_id': self.chunk_id,
'mtime': self.mtime,
'rtime': self.rtime}
def setUp(self):
super(TestRdirServer, self).setUp()
self.db_path = tempfile.mkdtemp()
self.conf = {'db_path': self.db_path, 'namespace': 'OPENIO'}
self.app = Client(create_app(self.conf), BaseResponse)
self.volume = 'testvolume'
self.app.get("/v1/rdir/create", query_string={'vol': self.volume})
self.container_id = random_id(64)
self.content_id = random_id(32)
self.chunk_id = random_id(64)
self.mtime = int(time.time())
self.rtime = 0
self.meta = {
'container_id': self.container_id,
'content_id': self.content_id,
'chunk_id': self.chunk_id,
'mtime': self.mtime,
'rtime': self.rtime
}
def setUp(self):
super(TestBlobAuditorFunctional, self).setUp()
self.namespace = self.conf['namespace']
self.account = self.conf['account']
self.test_dir = self.conf['sds_path']
rawx_num, rawx_path, rawx_addr = self.get_service_url('rawx')
self.rawx = 'http://' + rawx_addr
self.h = hashlib.new('md5')
conf = {"namespace": self.namespace}
self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
self.container_c = ContainerClient(conf)
self.blob_c = BlobClient()
self.ref = random_str(8)
self.container_c.container_create(self.account, self.ref)
self.url_rand = random_id(64)
self.data = random_str(1280)
self.h.update(self.data)
self.hash_rand = self.h.hexdigest().lower()
self.content = TestContent(
random_str(6), len(self.data), self.url_rand, 1)
self.content.id_container = cid_from_name(
self.account, self.ref).upper()
self.chunk = TestChunk(self.content.size, self.url_rand, 0,
self.hash_rand)
self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0",
"size": self.chunk.size,
"url": self.chunk_url}
chunk_meta = {'content_path': self.content.path,
'container_id': self.content.id_container,
'content_chunkmethod': 'plain/nb_copy=3',
'content_policy': 'TESTPOLICY',
'content_id': '0000',
'content_version': 1,
'chunk_id': self.chunk.id_chunk,
'chunk_pos': self.chunk.pos}
self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)
self.chunk_path = self.test_dir + '/data/' + self.namespace + \
'-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk
self.bad_container_id = '0'*64
def __init__(self, account, ref):
self.cid = cid_from_name(account, ref)
self.path = random_str(6)
self.version = 1
self.id = random_id(32)
self.fullpath = encode_fullpath(account, ref, self.path, self.version,
self.id)
self.data = os.urandom(1280)
self.size = len(self.data)
md5 = hashlib.new('md5')
md5.update(self.data)
self.hash = md5.hexdigest().lower()
def _create_chunk(self, rawx_path, alias="toto"):
container_id = random_id(64)
content_id = random_id(32)
chunk_id = random_id(64)
chunk_dir = "%s/%s" % (rawx_path, chunk_id[0:3])
if not os.path.isdir(chunk_dir):
os.makedirs(chunk_dir)
chunk_path = "%s/%s" % (chunk_dir, chunk_id)
with open(chunk_path, "w") as f:
f.write("toto")
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['chunk_hash'], 32 * '0')
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['chunk_id'], chunk_id)
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['chunk_pos'], '0')
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['chunk_size'], '4')
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['container_id'],
container_id)
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['content_id'], content_id)
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['content_path'], alias)
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['content_policy'],
'TESTPOLICY')
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['content_chunkmethod'],
'plain/nb_copy=3')
xattr.setxattr(
chunk_path, 'user.' + chunk_xattr_keys['content_version'], '0')
return chunk_path, container_id, content_id, chunk_id
def setUp(self):
super(TestRdirBackend, self).setUp()
self.db_path = tempfile.mkdtemp()
self.conf = {'db_path': self.db_path}
self.rdir = RdirBackend(self.conf)
self.volume = random_id(32)
self.rdir.create(self.volume)
self.container_0 = '0' + random_id(63)
self.container_1 = '1' + random_id(63)
self.container_2 = '2' + random_id(63)
self.content_0 = '0' + random_id(31)
self.content_1 = '1' + random_id(31)
self.chunk_0 = random_id(64)