예제 #1
0
 def __init__(self, namespace, endpoint, **kwargs):
     endpoint_v3 = '/'.join([endpoint.rstrip('/'), 'v3.0'])
     super(ObjectStorageAPI, self).__init__(endpoint=endpoint_v3, **kwargs)
     self.directory = DirectoryAPI(
         namespace,
         endpoint,
         session=self.session
     )
     self.namespace = namespace
예제 #2
0
    def setUp(self):
        super(TestDirectoryFunctional, self).setUp()

        self.reference_name = "func-test-reference-%s" % uuid.uuid4()
        self.reference_name_2 = "func-test-reference-%s-2" % uuid.uuid4()
        self.reference_name_3 = "func-test-reference-%s-3" % uuid.uuid4()

        self.directory = DirectoryAPI(self.namespace, self.proxyd_uri)

        self.directory.create(self.account, self.reference_name)
        self.directory.create(self.account, self.reference_name_2)
        self.directory.link(self.account, self.reference_name, "meta2")
예제 #3
0
class ObjectStorageAPI(API):
    """
    The Object Storage API
    """

    def __init__(self, namespace, endpoint, **kwargs):
        endpoint_v3 = '/'.join([endpoint.rstrip('/'), 'v3.0'])
        super(ObjectStorageAPI, self).__init__(endpoint=endpoint_v3, **kwargs)
        self.directory = DirectoryAPI(
            namespace,
            endpoint,
            session=self.session
        )
        self.namespace = namespace

    def account_create(self, account, headers=None):
        uri = '/v1.0/account/create'
        account_id = utils.quote(account, '')
        params = {'id': account_id}
        resp, resp_body = self._account_request('PUT', uri, params=params,
                                                headers=headers)
        created = (resp.status_code == 201)
        return created

    def account_show(self, account, headers=None):
        uri = "/v1.0/account/show"
        account_id = utils.quote(account, '')
        params = {'id': account_id}
        resp, resp_body = self._account_request('GET', uri, params=params,
                                                headers=headers)
        return resp_body

    def account_update(self, account, metadata, to_delete=None, headers=None):
        uri = "/v1.0/account/update"
        account_id = utils.quote(account, '')
        params = {'id': account_id}
        data = json.dumps({"metadata": metadata, "to_delete": to_delete})
        resp, resp_body = self._account_request('POST', uri, params=params,
                                                data=data, headers=headers)

    def account_set_properties(self, account, properties, headers=None):
        self.account_update(account, properties, headers=headers)

    def account_del_properties(self, account, properties, headers=None):
        self.account_update(account, None, properties, headers=headers)

    def container_create(self, account, container, metadata=None,
                         headers=None):
        uri = self._make_uri('container/create')
        params = self._make_params(account, container)

        headers = headers or {}
        headers['x-oio-action-mode'] = 'autocreate'
        if metadata:
            headers_meta = {}
            for k, v in metadata.iteritems():
                headers_meta['%suser-%s' % (CONTAINER_METADATA_PREFIX, k)] = v
            headers.update(headers_meta)
        resp, resp_body = self._request(
            'POST', uri, params=params, headers=headers)
        if resp.status_code not in (204, 201):
            raise exc.from_response(resp, resp_body)
        if resp.status_code == 201:
            return False
        else:
            return True

    @handle_container_not_found
    def container_delete(self, account, container, headers=None):
        uri = self._make_uri('container/destroy')
        params = self._make_params(account, container)
        try:
            resp, resp_body = self._request(
                'POST', uri, params=params, headers=headers)
        except exc.Conflict as e:
            raise exc.ContainerNotEmpty(e)

        self.directory.unlink(account, container, "meta2", headers=headers)

    def container_list(self, account, limit=None, marker=None,
                       end_marker=None, prefix=None, delimiter=None,
                       headers=None):
        uri = "v1.0/account/containers"
        account_id = utils.quote(account, '')
        params = {"id": account_id, "limit": limit, "marker": marker,
                  "delimiter": delimiter, "prefix": prefix,
                  "end_marker": end_marker}

        resp, resp_body = self._account_request(
            'GET', uri, params=params, headers=headers)
        listing = resp_body['listing']
        del resp_body['listing']
        return listing, resp_body

    @handle_container_not_found
    def container_show(self, account, container, headers=None):
        uri = self._make_uri('container/get_properties')
        params = self._make_params(account, container)
        resp, resp_body = self._request(
            'POST', uri, params=params, headers=headers)
        return resp_body

    def container_update(self, account, container, metadata, clear=False,
                         headers=None):
        if not metadata:
            self.container_del_properties(
                account, container, [], headers=headers)
        else:
            self.container_set_properties(
                account, container, metadata, clear, headers=headers)

    @handle_container_not_found
    def container_set_properties(self, account, container, properties,
                                 clear=False, headers=None):
        params = self._make_params(account, container)

        if clear:
            params.update({'flush': 1})

        uri = self._make_uri('container/set_properties')

        resp, resp_body = self._request(
            'POST', uri, data=json.dumps(properties), params=params,
            headers=headers)

    @handle_container_not_found
    def container_del_properties(self, account, container, properties,
                                 headers=None):
        params = self._make_params(account, container)

        uri = self._make_uri('container/del_properties')

        resp, resp_body = self._request(
            'POST', uri, data=json.dumps(properties), params=params,
            headers=headers)

    @handle_container_not_found
    def object_create(self, account, container, file_or_path=None, data=None,
                      etag=None, obj_name=None, content_type=None,
                      content_encoding=None, content_length=None,
                      metadata=None, policy=None, headers=None):
        if (data, file_or_path) == (None, None):
            raise exc.MissingData()
        src = data if data is not None else file_or_path
        if src is file_or_path:
            if isinstance(file_or_path, basestring):
                if not os.path.exists(file_or_path):
                    raise exc.FileNotFound("File '%s' not found." %
                                           file_or_path)
                file_name = os.path.basename(file_or_path)
            else:
                try:
                    file_name = os.path.basename(file_or_path.name)
                except AttributeError:
                    file_name = None
            obj_name = obj_name or file_name
        if not obj_name:
            raise exc.MissingName(
                "No name for the object has been specified"
            )

        if isinstance(data, basestring):
            content_length = len(data)

        if content_length is None:
            raise exc.MissingContentLength()

        sysmeta = {'mime_type': content_type,
                   'content_encoding': content_encoding,
                   'content_length': content_length,
                   'etag': etag}

        if src is data:
            return self._object_create(
                account, container, obj_name, StringIO(data), sysmeta,
                metadata=metadata, policy=policy, headers=headers)
        elif hasattr(file_or_path, "read"):
            return self._object_create(
                account, container, obj_name, src, sysmeta, metadata=metadata,
                policy=policy, headers=headers)
        else:
            with open(file_or_path, "rb") as f:
                return self._object_create(
                    account, container, obj_name, f, sysmeta,
                    metadata=metadata, policy=policy, headers=headers)

    @handle_object_not_found
    def object_delete(self, account, container, obj, headers=None):
        uri = self._make_uri('content/delete')
        params = self._make_params(account, container, obj)
        resp, resp_body = self._request(
            'POST', uri, params=params, headers=headers)

    @handle_container_not_found
    def object_list(self, account, container, limit=None, marker=None,
                    delimiter=None, prefix=None, end_marker=None,
                    include_metadata=False, headers=None):
        uri = self._make_uri('container/list')
        params = self._make_params(account, container)
        d = {"max": limit,
             "marker": marker,
             "delimiter": delimiter,
             "prefix": prefix,
             "end_marker": end_marker}
        params.update(d)

        resp, resp_body = self._request(
            'GET', uri, params=params, headers=headers)

        if include_metadata:
            metadata = {}
            for k, v in resp.headers.iteritems():
                if k.startswith('%suser-' % CONTAINER_METADATA_PREFIX):
                    metadata[k[len(CONTAINER_METADATA_PREFIX + 'user-'):]] = v
            return metadata, resp_body

        return resp_body

    @handle_object_not_found
    def object_analyze(self, account, container, obj, headers=None):
        uri = self._make_uri('content/show')
        params = self._make_params(account, container, obj)
        resp, resp_body = self._request(
            'GET', uri, params=params, headers=headers)
        meta = _make_object_metadata(resp.headers)
        raw_chunks = resp_body
        return meta, raw_chunks

    def object_fetch(self, account, container, obj, size=None, offset=0,
                     headers=None):
        meta, raw_chunks = self.object_analyze(
            account, container, obj, headers=headers)
        rain_security = len(raw_chunks[0]["pos"].split(".")) == 2
        chunks = _sort_chunks(raw_chunks, rain_security)
        stream = self._fetch_stream(
            meta, chunks, rain_security, size, offset, headers=headers)
        return meta, stream

    @handle_object_not_found
    def object_show(self, account, container, obj, headers=None):
        uri = self._make_uri('content/get_properties')
        params = self._make_params(account, container, obj)
        resp, resp_body = self._request(
            'POST', uri, params=params, headers=headers)

        meta = _make_object_metadata(resp.headers)
        meta['properties'] = resp_body
        return meta

    def object_update(self, account, container, obj, metadata, clear=False,
                      headers=None):
        if clear:
            self.object_del_properties(
                account, container, obj, [], headers=headers)
        if metadata:
            self.object_set_properties(
                account, container, obj, metadata, headers=headers)

    @handle_object_not_found
    def object_set_properties(self, account, container, obj, properties,
                              clear=False, headers=None):
        params = self._make_params(account, container, obj)
        if clear:
            params.update({'flush': 1})
        uri = self._make_uri('content/set_properties')
        resp, resp_body = self._request(
            'POST', uri, data=json.dumps(properties), params=params,
            headers=headers)

    @handle_object_not_found
    def object_del_properties(self, account, container, obj, properties,
                              headers=None):
        params = self._make_params(account, container, obj)
        uri = self._make_uri('content/del_properties')
        resp, resp_body = self._request(
            'POST', uri, data=json.dumps(properties), params=params,
            headers=headers)

    def _make_uri(self, action):
        uri = "%s/%s" % (self.namespace, action)
        return uri

    def _make_params(self, account, ref, obj=None):
        params = {'acct': account,
                  'ref': ref}
        if obj:
            params.update({'path': obj})
        return params

    def _get_account_url(self):
        uri = self._make_uri('lb/choose')
        params = {'pool': 'account'}
        resp, resp_body = self._request('GET', uri, params=params)
        if resp.status_code == 200:
            instance_info = resp_body[0]
            return 'http://%s/' % instance_info['addr']
        else:
            raise exc.ClientException(
                "could not find account instance url"
            )

    def _account_request(self, method, uri, **kwargs):
        account_url = self._get_account_url()
        resp, resp_body = self._request(method, uri, endpoint=account_url,
                                        **kwargs)
        return resp, resp_body

    def _content_prepare(self, account, container, obj_name, size,
                         policy=None, headers=None):
        uri = self._make_uri('content/prepare')
        params = self._make_params(account, container, obj_name)
        args = {'size': size}
        if policy:
            args['policy'] = policy
        headers = headers or {}
        headers['x-oio-action-mode'] = 'autocreate'
        resp, resp_body = self._request(
            'POST', uri, data=json.dumps(args), params=params,
            headers=headers)
        return resp.headers, resp_body

    def _content_create(self, account, container, obj_name, final_chunks,
                        headers=None):
        uri = self._make_uri('content/create')
        params = self._make_params(account, container, obj_name)
        data = json.dumps(final_chunks)
        resp, resp_body = self._request(
            'POST', uri, data=data, params=params, headers=headers)
        return resp.headers, resp_body

    def _object_create(self, account, container, obj_name, src,
                       sysmeta, metadata=None, policy=None, headers=None):
        meta, raw_chunks = self._content_prepare(
            account, container, obj_name, sysmeta['content_length'],
            policy=policy, headers=headers)

        sysmeta['id'] = meta[object_headers['id']]
        sysmeta['version'] = meta[object_headers['version']]
        sysmeta['policy'] = meta[object_headers['policy']]
        sysmeta['mime_type'] = meta[object_headers['mime_type']]
        sysmeta['chunk_method'] = meta[object_headers['chunk_method']]

        rain_security = len(raw_chunks[0]["pos"].split(".")) == 2
        if rain_security:
            raise exc.OioException('RAIN Security not supported.')

        chunks = _sort_chunks(raw_chunks, rain_security)
        final_chunks, bytes_transferred, content_checksum = self._put_stream(
            account, container, obj_name, src, sysmeta, chunks,
            headers=headers)

        sysmeta['etag'] = content_checksum

        hdrs = {}
        hdrs[object_headers['size']] = bytes_transferred
        hdrs[object_headers['hash']] = sysmeta['etag']
        hdrs[object_headers['version']] = sysmeta['version']
        hdrs[object_headers['id']] = sysmeta['id']
        hdrs[object_headers['policy']] = sysmeta['policy']
        hdrs[object_headers['mime_type']] = sysmeta['mime_type']
        hdrs[object_headers['chunk_method']] = sysmeta['chunk_method']

        if metadata:
            for k, v in metadata.iteritems():
                hdrs['%sx-%s' % (OBJECT_METADATA_PREFIX, k)] = v

        m, body = self._content_create(account, container, obj_name,
                                       final_chunks, headers=hdrs)
        return final_chunks, bytes_transferred, content_checksum

    def _put_stream(self, account, container, obj_name, src, sysmeta, chunks,
                    headers=None):
        global_checksum = hashlib.md5()
        total_bytes_transferred = 0
        content_chunks = []

        def _connect_put(chunk):
            raw_url = chunk["url"]
            parsed = urlparse(raw_url)
            try:
                chunk_path = parsed.path.split('/')[-1]
                hdrs = {}
                hdrs["transfer-encoding"] = "chunked"
                hdrs[chunk_headers["content_id"]] = sysmeta['id']
                hdrs[chunk_headers["content_version"]] = sysmeta['version']
                hdrs[chunk_headers["content_path"]] = utils.quote(obj_name)
                hdrs[chunk_headers["content_size"]] = sysmeta['content_length']
                hdrs[chunk_headers["content_chunkmethod"]] = \
                    sysmeta['chunk_method']
                hdrs[chunk_headers["content_mimetype"]] = sysmeta['mime_type']
                hdrs[chunk_headers["content_policy"]] = sysmeta['policy']
                hdrs[chunk_headers["content_chunksnb"]] = len(chunks)
                hdrs[chunk_headers["container_id"]] = \
                    utils.name2cid(account, container)
                hdrs[chunk_headers["chunk_pos"]] = chunk["pos"]
                hdrs[chunk_headers["chunk_id"]] = chunk_path
                with ConnectionTimeout(CONNECTION_TIMEOUT):
                    conn = http_connect(
                        parsed.netloc, 'PUT', parsed.path, hdrs)
                    conn.chunk = chunk
                return conn
            except (Exception, Timeout):
                pass

        def _send_data(conn):
            while True:
                data = conn.queue.get()
                if not conn.failed:
                    try:
                        with ChunkWriteTimeout(CHUNK_TIMEOUT):
                            conn.send(data)
                    except (Exception, ChunkWriteTimeout):
                        conn.failed = True
                conn.queue.task_done()

        for pos in range(len(chunks)):
            current_chunks = chunks[pos]

            pile = GreenPile(len(current_chunks))

            for current_chunk in current_chunks:
                pile.spawn(_connect_put, current_chunk)

            conns = [conn for conn in pile if conn]

            min_conns = 1

            if len(conns) < min_conns:
                raise exc.OioException("RAWX connection failure")

            bytes_transferred = 0
            total_size = current_chunks[0]["size"]
            chunk_checksum = hashlib.md5()
            try:
                with utils.ContextPool(len(current_chunks)) as pool:
                    for conn in conns:
                        conn.failed = False
                        conn.queue = Queue(PUT_QUEUE_DEPTH)
                        pool.spawn(_send_data, conn)

                    while True:
                        remaining_bytes = total_size - bytes_transferred
                        if WRITE_CHUNK_SIZE < remaining_bytes:
                            read_size = WRITE_CHUNK_SIZE
                        else:
                            read_size = remaining_bytes
                        with ChunkReadTimeout(CHUNK_TIMEOUT):
                            data = src.read(read_size)
                            if len(data) == 0:
                                for conn in conns:
                                    conn.queue.put('0\r\n\r\n')
                                break
                        chunk_checksum.update(data)
                        global_checksum.update(data)
                        bytes_transferred += len(data)
                        for conn in conns:
                            if not conn.failed:
                                conn.queue.put('%x\r\n%s\r\n' % (len(data),
                                                                 data))
                            else:
                                conns.remove(conn)

                        if len(conns) < min_conns:
                            raise exc.OioException("RAWX write failure")

                    for conn in conns:
                        if conn.queue.unfinished_tasks:
                            conn.queue.join()

                conns = [conn for conn in conns if not conn.failed]

            except ChunkReadTimeout:
                raise exc.ClientReadTimeout()
            except (Exception, Timeout):
                raise exc.OioException(
                    "Exception during chunk write."
                )

            final_chunks = []
            for conn in conns:
                resp = conn.getresponse(True)
                if resp.status in (200, 201):
                    conn.chunk["size"] = bytes_transferred
                    final_chunks.append(conn.chunk)
                conn.close()
            if len(final_chunks) < min_conns:
                raise exc.OioException("RAWX write failure")

            checksum = chunk_checksum.hexdigest()
            for chunk in final_chunks:
                chunk["hash"] = checksum
            content_chunks += final_chunks
            total_bytes_transferred += bytes_transferred

        content_checksum = global_checksum.hexdigest()

        return content_chunks, total_bytes_transferred, content_checksum

    def _fetch_stream(self, meta, chunks, rain_security, size, offset,
                      headers=None):
        current_offset = 0
        total_bytes = 0
        if size is None:
            size = int(meta["length"])
        if rain_security:
            raise exc.OioException("RAIN not supported")
        else:
            for pos in range(len(chunks)):
                chunk_size = int(chunks[pos][0]["size"])
                if total_bytes >= size:
                    break
                if current_offset + chunk_size > offset:
                    if current_offset < offset:
                        _offset = offset - current_offset
                    else:
                        _offset = 0
                    if chunk_size + total_bytes > size:
                        _size = size - total_bytes
                    else:
                        _size = chunk_size

                    handler = ChunkDownloadHandler(chunks[pos], _size, _offset)
                    stream = handler.get_stream()
                    if not stream:
                        raise exc.OioException("Error while downloading")
                    for s in stream:
                        total_bytes += len(s)
                        yield s
                current_offset += chunk_size
예제 #4
0
class TestDirectoryFunctional(FunctionalTestCase):
    def setUp(self):
        super(TestDirectoryFunctional, self).setUp()

        self.reference_name = "func-test-reference-%s" % uuid.uuid4()
        self.reference_name_2 = "func-test-reference-%s-2" % uuid.uuid4()
        self.reference_name_3 = "func-test-reference-%s-3" % uuid.uuid4()

        self.directory = DirectoryAPI(self.namespace, self.proxyd_uri)

        self.directory.create(self.account, self.reference_name)
        self.directory.create(self.account, self.reference_name_2)
        self.directory.link(self.account, self.reference_name, "meta2")

    def tearDown(self):
        super(TestDirectoryFunctional, self).tearDown()
        for ref in (self.reference_name, self.reference_name_2, self.reference_name_3):
            try:
                self.directory.delete(self.account, ref)
            except exceptions.ClientException:
                pass

    def test_has_reference(self):
        self.assertTrue(self.directory.has(self.account, self.reference_name))
        self.assertFalse(self.directory.has(self.account, self.reference_name_3))

    def test_stat_reference(self):
        reference = self.directory.get(self.account, self.reference_name)
        self.assertTrue(reference)

    def test_list_services_reference(self):
        services = self.directory.list_services(self.account, self.reference_name, "meta2")
        self.assertTrue(len(services))

    def test_create_reference(self):
        self.directory.create(self.account, self.reference_name_3)

    def test_delete_reference(self):
        self.directory.delete(self.account, self.reference_name_2)
        self.assertRaises(exceptions.NotFound, self.directory.get, self.account, self.reference_name_2)

    def test_link_reference(self):
        self.directory.link(self.account, self.reference_name_2, "meta2")
        services = self.directory.list_services(self.account, self.reference_name_2, "meta2")
        self.assertTrue(len(services["srv"]))

    def test_unlink_reference(self):
        self.directory.unlink(self.account, self.reference_name, "meta2")
        services = self.directory.list_services(self.account, self.reference_name, "meta2")
        self.assertFalse(len(services["srv"]))

    def test_renew_reference(self):
        services = self.directory.renew(self.account, self.reference_name, "meta2")
        self.assertTrue(len(services))

    def test_force_reference(self):
        services = {"seq": 2, "type": "meta2", "host": "127.0.0.1:7000", "args": ""}
        self.directory.force(self.account, self.reference_name, "meta2", services)

    def test_properties(self):
        self.directory.set_properties(self.account, self.reference_name, {"data": "something"})
        props = self.directory.get_properties(self.account, self.reference_name)
        self.assertTrue(props)
        self.assertEqual("something", props.get("data"))

        self.directory.del_properties(self.account, self.reference_name, ["data"])
        props = self.directory.get_properties(self.account, self.reference_name, ["data"])
        self.assertFalse(len(props))