Example #1
0
 def _record(self):
     return {
         "container_id": random_id(64),
         "content_id": random_id(32),
         "chunk_id": random_id(64),
         "mtime": int(time.time())
     }
Example #2
0
 def test_chunk_fetch_with_rebuild_container_id_start_after_limit(self):
     self.rdir.admin_incident_set(self.rawx_id, self.incident_date)
     self.rdir._direct_request.reset_mock()
     expected_entries_rebuild = [
         entry for entry in self.expected_entries if entry[2][-1] == '0'
     ]
     cid = random.choice(self.expected_entries)[0]
     expected_entries_rebuild_cid = \
         [entry for entry in expected_entries_rebuild
          if entry[0] == cid]
     if expected_entries_rebuild_cid:
         start_after_index = random.randrange(
             0, len(expected_entries_rebuild_cid))
         start_after = '|'.join(
             expected_entries_rebuild_cid[start_after_index][:3])
     else:
         start_after_index = 0
         start_after = '|'.join(
             (random_id(64), random_id(32), random_id(64)))
     entries = self.rdir.chunk_fetch(self.rawx_id,
                                     rebuild=True,
                                     container_id=cid,
                                     start_after=start_after,
                                     limit=2)
     self._assert_chunk_fetch(
         expected_entries_rebuild_cid[start_after_index + 1:],
         entries,
         limit=2)
Example #3
0
 def setUp(self) -> None:
     # Given
     self.test_type = random_id()
     self.test_id = random_id()
     self.test_uuid = random_uuid()
     self.test_case = make_ingest_entity(self.test_type, self.test_id, self.test_uuid)
     self.submission = HcaSubmission()
     # When
     self.test_entity = self.submission.map_ingest_entity(self.test_case)
Example #4
0
    def _create_chunk(self, rawx_path, alias="toto", suffix=''):
        cname = random_str(8)
        container_id = cid_from_name(self.account, cname)
        content_id = random_id(32)
        chunk_id = random_id(64)

        chunk_dir = "%s/%s" % (rawx_path, chunk_id[0:3])
        if not os.path.isdir(chunk_dir):
            os.makedirs(chunk_dir)

        chunk_path = "%s/%s%s" % (chunk_dir, chunk_id, suffix)
        with open(chunk_path, "w") as chunk_file:
            chunk_file.write("toto")

        # pylint: disable=no-member
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['chunk_hash'], 32 * b'0')
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['chunk_id'], enc(chunk_id))
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['chunk_pos'], b'0')
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['chunk_size'], b'4')
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['content_policy'],
            b'TESTPOLICY')
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['content_chunkmethod'],
            b'plain/nb_copy=3')
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['content_version'], b'1')
        # Old (oio-sds < 4.2) extended attributes
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['container_id'],
            enc(container_id))
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['content_id'],
            enc(content_id))
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['content_path'], enc(alias))
        # New (oio-sds >= 4.2) extended attributes
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['oio_version'],
            enc(OIO_VERSION))
        fullpath = encode_fullpath(self.account, cname, alias, 1, content_id)
        xattr.setxattr(
            chunk_path,
            'user.%s%s' % (CHUNK_XATTR_CONTENT_FULLPATH_PREFIX, chunk_id),
            enc(fullpath))

        return chunk_path, container_id, content_id, chunk_id
Example #5
0
 def setUp(self):
     self.endpoint = "http://1.2.3.4:8000"
     self.api = FakeDirectoryAPI("NS", self.endpoint)
     self.account = "AUTH_test"
     self.headers = {"x-req-id": random_id(64)}
     self.uri_base = "NS"
     self.name = random_str(64)
Example #6
0
 def __init__(self, rawx_id, rawx_loc, metachunk_size, metachunk_hash):
     self.id = random_id(64)
     self.url = "%s/%s" % (rawx_id, self.id)
     self.path = rawx_loc + '/' + self.id[0:3] + '/' + self.id
     self.pos = 0
     self.metachunk_size = metachunk_size
     self.metachunk_hash = metachunk_hash
Example #7
0
    def test_rdir_clear_and_lock(self):
        rec = self._record()
        who = random_id(32)

        # push with autocreate
        resp = self._post(
                "/v1/rdir/push", params={'vol': self.vol, 'create': True},
                data=json.dumps(rec))
        self.assertEqual(resp.status_code, 204)

        # lock
        resp = self._post(
                "/v1/rdir/admin/lock", params={'vol': self.vol},
                data=json.dumps({'who': who}))
        self.assertEqual(resp.status_code, 204)

        # try to clear while the lock is held
        resp = self._post("/v1/rdir/admin/clear", params={'vol': self.vol})
        self.assertEqual(resp.status_code, 403)

        # unlock
        resp = self._post("/v1/rdir/admin/unlock", params={'vol': self.vol})
        self.assertEqual(resp.status_code, 204)

        # clear all entries
        resp = self._post(
                "/v1/rdir/admin/clear", params={'vol': self.vol},
                data=json.dumps({'all': True}))
        self.assertEqual(resp.status_code, 200)
        self.assertEqual(resp.json(), {'removed': 1})
Example #8
0
    def test_rdir_clear_with_repair(self):
        rec = self._record()
        who = random_id(32)

        # clear without volume
        resp = self._post("/v1/rdir/admin/clear", params={'repair': True})
        self.assertEqual(resp.status, 400)

        # push with autocreate
        resp = self._post(
                "/v1/rdir/push", params={'vol': self.vol, 'create': True},
                data=json.dumps(rec))
        self.assertEqual(resp.status, 204)

        # lock
        resp = self._post(
                "/v1/rdir/admin/lock", params={'vol': self.vol},
                data=json.dumps({'who': who}))
        self.assertEqual(resp.status, 204)

        # try to clear while the lock is held
        resp = self._post("/v1/rdir/admin/clear",
                          params={'vol': self.vol, 'repair': True})
        self.assertEqual(resp.status, 403)

        # unlock
        resp = self._post("/v1/rdir/admin/unlock", params={'vol': self.vol})
        self.assertEqual(resp.status, 204)

        # repair all entries
        resp = self._post("/v1/rdir/admin/clear",
                          params={'vol': self.vol, 'repair': True})
        self.assertEqual(resp.status, 200)
        self.assertDictEqual(self.json_loads(resp.data),
                             {'removed': 0, 'repaired': 1, 'errors': 0})
Example #9
0
    def test_send_all_samples_sends_each_samples_with_project_release_date(self):
        # Given
        self.submitter.send_sample = MagicMock(return_value='UPDATED')
        test_date = datetime.now().isoformat()
        project_attributes = {
            'releaseDate': test_date
        }
        project_attributes = make_ingest_entity('projects', random_id(), random_uuid(), project_attributes)
        self.submission.map_ingest_entity(project_attributes)
        biomaterials = self.map_random_biomaterials(self.submission, 5)
        calls = []
        call_count = 0
        expected_result = {'UPDATED': []}
        for entity in biomaterials:
            call_count += 1
            calls.append(call(entity, test_date))
            expected_result['UPDATED'].append(entity)

        # When
        result = self.submitter.send_all_samples(self.submission)

        # Then
        self.submitter.send_sample.assert_has_calls(calls, any_order=True)
        self.assertEqual(call_count, self.submitter.send_sample.call_count)
        self.assertDictEqual(expected_result, result)
Example #10
0
 def test_bad_routes(self):
     routes = ('/status', '/config',
               '/v1/status',
               '/v1/rdir/admin/show',
               '/v1/rdir/admin/lock',
               '/v1/rdir/admin/unlock',
               '/v1/rdir/admin/incident',
               '/v1/rdir/admin/clear',
               '/v1/rdir/create',
               '/v1/rdir/push',
               '/v1/rdir/delete',
               '/v1/rdir/fetch',
               '/v1/rdir/status',
               '/v1/rdir/meta2/create',
               '/v1/rdir/meta2/push',
               '/v1/rdir/meta2/delete',
               '/v1/rdir/meta2/fetch')
     for r in routes:
         resp = self._get('/' + r)
         self.assertEqual(resp.status, 404)
         resp = self._get(r + '/')
         self.assertEqual(resp.status, 404)
         bulk = random_id(4)
         resp = self._get('/' + bulk + r)
         self.assertEqual(resp.status, 404)
         resp = self._get(r + '/' + bulk)
         self.assertEqual(resp.status, 404)
         resp = self._get(r + bulk)
         self.assertEqual(resp.status, 404)
Example #11
0
 def setUp(self):
     self.endpoint = "http://1.2.3.4:8000"
     self.api = FakeDirectoryAPI("NS", self.endpoint)
     self.account = "AUTH_test"
     self.headers = {"x-req-id": random_id(64)}
     self.uri_base = "NS"
     self.name = random_str(64)
Example #12
0
    def test_copy_with_same_chunkid(self):
        metachunk_hash = md5().hexdigest()
        trailers = {
            'x-oio-chunk-meta-metachunk-size': 1,
            'x-oio-chunk-meta-metachunk-hash': metachunk_hash
        }

        chunkid1 = random_chunk_id()
        chunkdata1 = random_buffer(string.printable, 1)
        chunkurl1 = self._rawx_url(chunkid1)
        headers1 = self._chunk_attr(chunkid1, chunkdata1)
        self._check_not_present(chunkurl1)
        resp, _ = self._http_request(chunkurl1, 'PUT', chunkdata1, headers1,
                                     trailers)
        self.assertEqual(201, resp.status)
        self.assertEqual(headers1['x-oio-chunk-meta-chunk-hash'].upper(),
                         resp.getheader('x-oio-chunk-meta-chunk-hash'))
        self.assertEqual(headers1['x-oio-chunk-meta-chunk-size'].upper(),
                         resp.getheader('x-oio-chunk-meta-chunk-size'))

        headers = {}
        headers["Destination"] = chunkurl1
        headers['x-oio-chunk-meta-full-path'] = encode_fullpath(
            "account-snapshot", "container-snapshot", "content-snapshot",
            1456938361143741, random_id(32))
        resp, _ = self._http_request(chunkurl1, 'COPY', '', headers)
        self.assertEqual(403, resp.status)
Example #13
0
 def test_admin_show(self):
     who = random_id(32)
     self.rdir.admin_lock(self.volume, who)
     self.rdir.admin_set_incident_date(self.volume, 1234)
     res = self.rdir.admin_show(self.volume)
     self.assertEqual(res, {'incident_date': "1234",
                            'lock': who})
Example #14
0
 def _put_chunk(self):
     account = random_str(16)
     container = random_str(16)
     cid = cid_from_name(account, container)
     content_path = random_str(16)
     content_version = 1234567890
     content_id = random_id(32)
     fullpath = encode_fullpath(account, container, content_path,
                                content_version, content_id)
     chunk_id = random_chunk_id()
     data = random_buffer(string.printable, 100)
     meta = {
         'full_path': fullpath,
         'container_id': cid,
         'content_path': content_path,
         'version': content_version,
         'id': content_id,
         'chunk_method': 'ec/algo=liberasurecode_rs_vand,k=6,m=3',
         'policy': 'TESTPOLICY',
         'chunk_hash': md5(data).hexdigest().upper(),
         'oio_version': OIO_VERSION,
         'chunk_pos': 0,
         'metachunk_hash': md5().hexdigest(),
         'metachunk_size': 1024
     }
     self.blob_client.chunk_put('http://' + self.rawx_id + '/' + chunk_id,
                                meta, data)
     sleep(1)  # ensure chunk event have been processed
     return account, container, cid, content_path, content_version, \
         content_id, chunk_id
Example #15
0
    def test_copy_with_nonexistent_source(self):
        metachunk_hash = md5().hexdigest()
        trailers = {'x-oio-chunk-meta-metachunk-size': '1',
                    'x-oio-chunk-meta-metachunk-hash': metachunk_hash}

        chunkid1 = random_chunk_id()
        chunkurl1 = self._rawx_url(chunkid1)

        chunkid2 = random_chunk_id()
        chunkdata2 = random_buffer(string.printable, 1).encode('utf-8')
        chunkurl2 = self._rawx_url(chunkid2)
        headers2 = self._chunk_attr(chunkid2, chunkdata2)
        self._check_not_present(chunkurl2)
        resp, _ = self._http_request(chunkurl2, 'PUT', chunkdata2, headers2,
                                     trailers)
        self.assertEqual(201, resp.status)
        self.assertEqual(headers2['x-oio-chunk-meta-chunk-hash'].upper(),
                         resp.getheader('x-oio-chunk-meta-chunk-hash'))
        self.assertEqual(headers2['x-oio-chunk-meta-chunk-size'],
                         resp.getheader('x-oio-chunk-meta-chunk-size'))

        headers = {}
        headers["Destination"] = chunkurl2
        headers['x-oio-chunk-meta-full-path'] = encode_fullpath(
                "account-snapshot", "container-snapshot", "content-snapshot",
                1456938361143741, random_id(32))
        resp, _ = self._http_request(chunkurl1, 'COPY', '', headers)
        self.assertEqual(404, resp.status)
Example #16
0
    def _create_content(self, name):
        headers = {'X-oio-action-mode': 'autocreate'}
        params = self.param_content(self.ref, name)
        resp = self.request('POST',
                            self.url_content('prepare'),
                            params=params,
                            headers=headers,
                            data=json.dumps({'size': '1024'}))
        self.assertEqual(resp.status, 200)
        chunks = self.json_loads(resp.data)

        stgpol = resp.getheader('x-oio-content-meta-policy')
        headers = {
            'x-oio-action-mode': 'autocreate',
            'x-oio-content-meta-size': '1024',
            'x-oio-content-meta-policy': stgpol,
            'x-oio-content-meta-version': int(time.time() * 1000000),
            'x-oio-content-meta-id': random_id(32)
        }
        resp = self.request('POST',
                            self.url_content('create'),
                            params=params,
                            headers=headers,
                            data=json.dumps(chunks))
        self.assertEqual(resp.status, 204)
Example #17
0
    def setUp(self):
        super(TestBlobAuditorFunctional, self).setUp()
        self.namespace = self.conf['namespace']
        self.account = self.conf['account']

        self.test_dir = self.conf['sds_path']

        rawx_num, rawx_path, rawx_addr = self.get_service_url('rawx')
        self.rawx = 'http://' + rawx_addr

        self.h = hashlib.new('md5')

        conf = {"namespace": self.namespace}
        self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
        self.container_c = ContainerClient(conf)
        self.blob_c = BlobClient()

        self.ref = random_str(8)

        self.container_c.container_create(self.account, self.ref)

        self.url_rand = random_id(64)

        self.data = random_str(1280)
        self.h.update(self.data)
        self.hash_rand = self.h.hexdigest().lower()

        self.content = TestContent(random_str(6), len(self.data),
                                   self.url_rand, 1)

        self.content.id_container = cid_from_name(self.account,
                                                  self.ref).upper()
        self.chunk = TestChunk(self.content.size, self.url_rand, 0,
                               self.hash_rand)

        self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
        self.chunk_proxy = {
            "hash": self.chunk.md5,
            "pos": "0",
            "size": self.chunk.size,
            "url": self.chunk_url
        }

        chunk_meta = {
            'content_path': self.content.path,
            'container_id': self.content.id_container,
            'chunk_method': 'plain/nb_copy=3',
            'policy': 'TESTPOLICY',
            'id': '0000',
            'version': 1,
            'chunk_id': self.chunk.id_chunk,
            'chunk_pos': self.chunk.pos,
            'chunk_hash': self.chunk.md5,
        }
        self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)

        self.chunk_path = self.test_dir + '/data/' + self.namespace + \
            '-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk
        self.bad_container_id = '0' * 64
Example #18
0
 def map_random_entities(
         submission: HcaSubmission,
         entity_count: int,
         entity_type: str = None
 ):
     entity_list = []
     for i in range(0, entity_count):
         entity_list.append(
             submission.map_ingest_entity(
                 make_ingest_entity(
                     entity_type if entity_type else random_id(),
                     random_id(),
                     random_uuid()
                 )
             )
         )
     return entity_list
Example #19
0
 def setUp(self):
     self.endpoint = "http://1.2.3.4:8000"
     self.api = FakeDirectoryClient({'namespace': "NS"},
                                    endpoint=self.endpoint)
     self.account = "AUTH_test"
     self.headers = {"x-req-id": random_id(64)}
     self.uri_base = "%s/v3.0/NS" % self.endpoint
     self.name = random_str(64)
Example #20
0
    def setUp(self):
        super(TestBlobAuditorFunctional, self).setUp()
        self.namespace = self.conf["namespace"]
        self.account = self.conf["account"]

        self.test_dir = self.conf["sds_path"]

        rawx_num, rawx_path, rawx_addr = self.get_service_url("rawx")
        self.rawx = "http://" + rawx_addr

        self.h = hashlib.new("md5")

        conf = {"namespace": self.namespace}
        self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
        self.container_c = ContainerClient(conf)
        self.blob_c = BlobClient()

        self.ref = random_str(8)

        self.container_c.container_create(self.account, self.ref)

        self.url_rand = random_id(64)

        self.data = random_str(1280)
        self.h.update(self.data)
        self.hash_rand = self.h.hexdigest().lower()

        self.content = TestContent(random_str(6), len(self.data), self.url_rand, 1)

        self.content.id_container = cid_from_name(self.account, self.ref).upper()
        self.chunk = TestChunk(self.content.size, self.url_rand, 0, self.hash_rand)

        self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
        self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0", "size": self.chunk.size, "url": self.chunk_url}

        chunk_meta = {
            "content_path": self.content.path,
            "container_id": self.content.id_container,
            "chunk_method": "plain/nb_copy=3",
            "policy": "TESTPOLICY",
            "id": "0000",
            "version": 1,
            "chunk_id": self.chunk.id_chunk,
            "chunk_pos": self.chunk.pos,
            "chunk_hash": self.chunk.md5,
        }
        self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)

        self.chunk_path = (
            self.test_dir
            + "/data/"
            + self.namespace
            + "-rawx-1/"
            + self.chunk.id_chunk[0:3]
            + "/"
            + self.chunk.id_chunk
        )
        self.bad_container_id = "0" * 64
Example #21
0
 def setUp(self):
     super(TestRdirClient, self).setUp()
     self.namespace = self.conf["namespace"]
     self.rdir_client = RdirClient({"namespace": self.namespace})
     self.rdir_client._get_rdir_addr = Mock(return_value="0.1.2.3:4567")
     self.container_id_1 = random_id(64)
     self.container_id_2 = random_id(64)
     self.container_id_3 = random_id(64)
     self.content_id_1 = random_id(32)
     self.content_id_2 = random_id(32)
     self.content_id_3 = random_id(32)
     self.chunk_id_1 = random_id(64)
     self.chunk_id_2 = random_id(64)
     self.chunk_id_3 = random_id(64)
Example #22
0
 def setUp(self):
     super(TestRdirClient, self).setUp()
     self.namespace = self.conf['namespace']
     self.rdir_client = RdirClient({'namespace': self.namespace})
     self.rdir_client._get_rdir_addr = Mock(return_value="0.1.2.3:4567")
     self.container_id_1 = random_id(64)
     self.container_id_2 = random_id(64)
     self.container_id_3 = random_id(64)
     self.content_id_1 = random_id(32)
     self.content_id_2 = random_id(32)
     self.content_id_3 = random_id(32)
     self.chunk_id_1 = random_id(64)
     self.chunk_id_2 = random_id(64)
     self.chunk_id_3 = random_id(64)
Example #23
0
 def map_random_biomaterials(submission: HcaSubmission, entity_count: int) -> List[Entity]:
     biomaterials = []
     for i in range(0, entity_count):
         biomaterials.append(
             submission.map_ingest_entity(
                 make_ingest_entity('biomaterials', random_id(), random_uuid())
             )
         )
     return biomaterials
Example #24
0
    def test_attributes_set_correctly_when_biosamples_accession_is_added_to_attributes(self):
        # When
        random_accession = random_id()
        self.test_entity.add_accession('BioSamples', random_accession)
        self.submission.add_accessions_to_attributes(self.test_entity)

        # Then
        acc = self.test_entity.attributes['content']['biomaterial_core']['biosamples_accession']
        self.assertEqual(random_accession, acc)
Example #25
0
 def setUp(self):
     super(TestRdirClient, self).setUp()
     self.namespace = "dummy"
     self.rdir_client = RdirClient({'namespace': self.namespace},
                                   endpoint='127.0.0.0:6000')
     self.rdir_client._get_rdir_addr = Mock(return_value="0.1.2.3:4567")
     self.container_id_1 = random_id(64)
     self.container_id_2 = random_id(64)
     self.container_id_3 = random_id(64)
     self.content_id_1 = random_id(32)
     self.content_id_2 = random_id(32)
     self.content_id_3 = random_id(32)
     self.chunk_id_1 = random_id(64)
     self.chunk_id_2 = random_id(64)
     self.chunk_id_3 = random_id(64)
Example #26
0
    def test_admin_lock_unlock(self):
        # unlock without lock
        self.rdir.admin_unlock(self.volume)

        # lock
        who = random_id(32)
        res = self.rdir.admin_lock(self.volume, who)
        self.assertEqual(res, None)

        # double lock
        res = self.rdir.admin_lock(self.volume, random_id(32))
        self.assertEqual(res, who)

        # unlock
        self.rdir.admin_unlock(self.volume)

        # lock again
        res = self.rdir.admin_lock(self.volume, random_id(32))
        self.assertEqual(res, None)
Example #27
0
 def _meta2_record(self):
     return {
         "container_id":
         random_id(64),
         "container_url":
         "{0}/{1}/{2}".format(random_str(12), random_str(12),
                              random_str(12)),
         "mtime":
         int(time.time())
     }
Example #28
0
    def test_cycle_content(self):
        path = random_content()
        headers = {'x-oio-action-mode': 'autocreate'}
        params = self.param_content(self.ref, path)

        resp = self.request('GET', self.url_content('show'), params=params)
        self.assertError(resp, 404, 406)

        resp = self.request('POST', self.url_content('touch'), params=params)
        self.assertError(resp, 404, 406)

        resp = self.request('POST',
                            self.url_content('prepare'),
                            data=json.dumps({'size': '1024'}),
                            params=params,
                            headers=headers)
        self.assertEqual(resp.status, 200)
        chunks = self.json_loads(resp.data)

        stgpol = resp.getheader('x-oio-content-meta-policy')
        headers = {
            'x-oio-action-mode': 'autocreate',
            'x-oio-content-meta-size': '1024',
            'x-oio-content-meta-policy': stgpol,
            'x-oio-content-meta-version': int(time.time() * 1000000),
            'x-oio-content-meta-id': random_id(32)
        }
        resp = self.request('POST',
                            self.url_content('create'),
                            params=params,
                            headers=headers,
                            data=json.dumps(chunks))
        self.assertEqual(resp.status, 204)

        # # FIXME check re-create depending on the container's ver'policy
        # resp = self.request('POST', self.url_content('create'),
        #                         params=params,
        #                         headers=headers,
        #                         data=json.dumps(chunks))
        # self.assertEqual(resp.status, 201)

        resp = self.request('GET', self.url_content('show'), params=params)
        self.assertEqual(resp.status, 200)

        resp = self.request('GET', self.url_content('show'), params=params)
        self.assertEqual(resp.status, 200)

        resp = self.request('POST', self.url_content('delete'), params=params)
        self.assertEqual(resp.status, 204)

        resp = self.request('GET', self.url_content('show'), params=params)
        self.assertError(resp, 404, 420)

        resp = self.request('POST', self.url_content('delete'), params=params)
        self.assertError(resp, 404, 420)
Example #29
0
 def test_submit_sample_return_error(self):
     # Given
     error_msg = f'TestCaseError{random_id(3)}'
     self.biosamples.send_sample = MagicMock(side_effect=Exception(error_msg))
     test_case = self.submission.map_ingest_entity(
         make_ingest_entity('biomaterials', random_id(), random_uuid()))
     # When
     result = self.submitter.send_sample(test_case)
     # Then
     self.assertEqual('ERRORED', result)
     self.assertDictEqual(test_case.get_errors(), {'content.biomaterial_core.biosamples_accession': [f'BioSamples Error: {error_msg}']})
Example #30
0
    def test_worm_and_admin(self):
        if not os.getenv("WORM"):
            self.skipTest("must be in worm mode")
        data = random_data(10)
        path = 'test_worm'
        content = self._new_content(data, path, True)

        # Prepare without admin mode
        self.assertRaises(Conflict, self._prepare_content, path, None, False)
        self.assertRaises(Conflict, self._prepare_content, path,
                          content.content_id, False)
        self.assertRaises(Conflict, self._prepare_content, 'test_worm_prepare',
                          content.content_id, False)
        self.assertRaises(Conflict, self._prepare_content, path, random_id(32),
                          False)

        # Overwrite without admin mode
        data2 = random_data(11)
        self.assertRaises(Conflict, self._new_content, data2, path, False)

        # Prepare with admin mode
        self._prepare_content(path, None, True)
        self._prepare_content(path, content.content_id, True)
        self._prepare_content('test_worm_prepare', content.content_id, True)
        self._prepare_content(path, random_id(32), True)

        # Overwrite with admin mode
        content = self._new_content(data2, path, True)

        # Delete without admin mode
        try:
            content.delete()
        except ClientException as exc:
            self.assertIn('worm', str(exc))
        else:
            self.fail("Delete without admin mode: no exception")
        downloaded_data = ''.join(content.fetch())
        self.assertEqual(downloaded_data, data2)

        # Delete with admin mode
        content.delete(admin_mode=True)
Example #31
0
    def test_worm_and_admin(self):
        if not os.getenv("WORM"):
            self.skipTest("must be in worm mode")
        data = random_data(10)
        path = 'test_worm'
        content = self._new_content(data, path, True)

        # Prepare without admin mode:
        # Since the 'prepare' step is done in the proxy, there is no check
        # on the pre-existence of the content. The subsequent prepare MUST
        # now work despite the presence of the content.
        self._prepare_content(path, None, False)
        self._prepare_content(path, content.content_id, False)
        self._prepare_content('test_worm_prepare', content.content_id, False)
        self._prepare_content(path, random_id(32), False)

        # Overwrite without admin mode
        data2 = random_data(11)
        self.assertRaises(Conflict, self._new_content, data2, path, False)

        # Prepare with admin mode
        self._prepare_content(path, None, True)
        self._prepare_content(path, content.content_id, True)
        self._prepare_content('test_worm_prepare', content.content_id, True)
        self._prepare_content(path, random_id(32), True)

        # Overwrite with admin mode
        content = self._new_content(data2, path, True)

        # Delete without admin mode
        try:
            content.delete()
        except ClientException as exc:
            self.assertIn('worm', str(exc))
        else:
            self.fail("Delete without admin mode: no exception")
        downloaded_data = ''.join(content.fetch())
        self.assertEqual(downloaded_data, data2)

        # Delete with admin mode
        content.delete(admin_mode=True)
Example #32
0
    def setUp(self):
        super(TestRdirServer, self).setUp()

        self.db_path = tempfile.mkdtemp()
        self.conf = {'db_path': self.db_path,
                     'namespace': 'OPENIO'}

        self.app = Client(create_app(self.conf), BaseResponse)
        self.volume = 'testvolume'
        self.app.get("/v1/rdir/create", query_string={'vol': self.volume})
        self.container_id = random_id(64)
        self.content_id = random_id(32)
        self.chunk_id = random_id(64)
        self.mtime = int(time.time())
        self.rtime = 0
        self.meta = {
            'container_id': self.container_id,
            'content_id': self.content_id,
            'chunk_id': self.chunk_id,
            'mtime': self.mtime,
            'rtime': self.rtime}
Example #33
0
    def setUp(self):
        super(TestRdirServer, self).setUp()

        self.db_path = tempfile.mkdtemp()
        self.conf = {'db_path': self.db_path, 'namespace': 'OPENIO'}

        self.app = Client(create_app(self.conf), BaseResponse)
        self.volume = 'testvolume'
        self.app.get("/v1/rdir/create", query_string={'vol': self.volume})
        self.container_id = random_id(64)
        self.content_id = random_id(32)
        self.chunk_id = random_id(64)
        self.mtime = int(time.time())
        self.rtime = 0
        self.meta = {
            'container_id': self.container_id,
            'content_id': self.content_id,
            'chunk_id': self.chunk_id,
            'mtime': self.mtime,
            'rtime': self.rtime
        }
    def setUp(self):
        super(TestBlobAuditorFunctional, self).setUp()
        self.namespace = self.conf['namespace']
        self.account = self.conf['account']

        self.test_dir = self.conf['sds_path']

        rawx_num, rawx_path, rawx_addr = self.get_service_url('rawx')
        self.rawx = 'http://' + rawx_addr

        self.h = hashlib.new('md5')

        conf = {"namespace": self.namespace}
        self.auditor = BlobAuditorWorker(conf, get_logger(None), None)
        self.container_c = ContainerClient(conf)
        self.blob_c = BlobClient()

        self.ref = random_str(8)

        self.container_c.container_create(self.account, self.ref)

        self.url_rand = random_id(64)

        self.data = random_str(1280)
        self.h.update(self.data)
        self.hash_rand = self.h.hexdigest().lower()

        self.content = TestContent(
            random_str(6), len(self.data), self.url_rand, 1)

        self.content.id_container = cid_from_name(
            self.account, self.ref).upper()
        self.chunk = TestChunk(self.content.size, self.url_rand, 0,
                               self.hash_rand)

        self.chunk_url = "%s/%s" % (self.rawx, self.chunk.id_chunk)
        self.chunk_proxy = {"hash": self.chunk.md5, "pos": "0",
                            "size": self.chunk.size,
                            "url":  self.chunk_url}

        chunk_meta = {'content_path': self.content.path,
                      'container_id': self.content.id_container,
                      'content_chunkmethod': 'plain/nb_copy=3',
                      'content_policy': 'TESTPOLICY',
                      'content_id': '0000',
                      'content_version': 1,
                      'chunk_id': self.chunk.id_chunk,
                      'chunk_pos': self.chunk.pos}
        self.blob_c.chunk_put(self.chunk_url, chunk_meta, self.data)

        self.chunk_path = self.test_dir + '/data/' + self.namespace + \
            '-rawx-1/' + self.chunk.id_chunk[0:3] + "/" + self.chunk.id_chunk
        self.bad_container_id = '0'*64
Example #35
0
 def __init__(self, account, ref):
     self.cid = cid_from_name(account, ref)
     self.path = random_str(6)
     self.version = 1
     self.id = random_id(32)
     self.fullpath = encode_fullpath(account, ref, self.path, self.version,
                                     self.id)
     self.data = os.urandom(1280)
     self.size = len(self.data)
     md5 = hashlib.new('md5')
     md5.update(self.data)
     self.hash = md5.hexdigest().lower()
Example #36
0
    def _create_chunk(self, rawx_path, alias="toto"):
        container_id = random_id(64)
        content_id = random_id(32)
        chunk_id = random_id(64)

        chunk_dir = "%s/%s" % (rawx_path, chunk_id[0:3])
        if not os.path.isdir(chunk_dir):
            os.makedirs(chunk_dir)

        chunk_path = "%s/%s" % (chunk_dir, chunk_id)
        with open(chunk_path, "w") as f:
            f.write("toto")

        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['chunk_hash'], 32 * '0')
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['chunk_id'], chunk_id)
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['chunk_pos'], '0')
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['chunk_size'], '4')
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['container_id'],
            container_id)
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['content_id'], content_id)
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['content_path'], alias)
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['content_policy'],
            'TESTPOLICY')
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['content_chunkmethod'],
            'plain/nb_copy=3')
        xattr.setxattr(
            chunk_path, 'user.' + chunk_xattr_keys['content_version'], '0')

        return chunk_path, container_id, content_id, chunk_id
Example #37
0
 def setUp(self):
     super(TestRdirBackend, self).setUp()
     self.db_path = tempfile.mkdtemp()
     self.conf = {'db_path': self.db_path}
     self.rdir = RdirBackend(self.conf)
     self.volume = random_id(32)
     self.rdir.create(self.volume)
     self.container_0 = '0' + random_id(63)
     self.container_1 = '1' + random_id(63)
     self.container_2 = '2' + random_id(63)
     self.content_0 = '0' + random_id(31)
     self.content_1 = '1' + random_id(31)
     self.chunk_0 = random_id(64)