예제 #1
0
 def test_from_response_http_status(self):
     fake_resp = FakeAPIResponse()
     fake_resp.status_code = 404
     exc = exceptions.from_response(fake_resp, None)
     self.assertTrue(isinstance(exc, exceptions.NotFound))
     fake_resp.status_code = 409
     exc = exceptions.from_response(fake_resp, None)
     self.assertTrue(isinstance(exc, exceptions.Conflict))
예제 #2
0
 def test_from_response_http_status(self):
     fake_resp = FakeApiResponse()
     fake_resp.status = 404
     exc = exceptions.from_response(fake_resp, None)
     self.assertIsInstance(exc, exceptions.NotFound)
     fake_resp.status = 409
     exc = exceptions.from_response(fake_resp, None)
     self.assertIsInstance(exc, exceptions.Conflict)
예제 #3
0
    def container_create(self,
                         account,
                         reference,
                         properties=None,
                         system=None,
                         **kwargs):
        """
        Create a container.

        :param account: account in which to create the container
        :type account: `str`
        :param reference: name of the container
        :type reference: `str`
        :param properties: properties to set on the container
        :type properties: `dict`
        :param system: system properties to set on the container
        :type system: `dict`
        :keyword headers: extra headers to send to the proxy
        :type headers: `dict`
        :returns: True if the container has been created,
                  False if it already exists
        """
        params = self._make_params(account, reference)
        data = json.dumps({
            'properties': properties or {},
            'system': system or {}
        })
        resp, body = self._request('POST',
                                   '/create',
                                   params=params,
                                   data=data,
                                   **kwargs)
        if resp.status not in (204, 201):
            raise exceptions.from_response(resp, body)
        return resp.status == 201
예제 #4
0
 def chunk_put(self, url, meta, data, **kwargs):
     headers = gen_put_headers(meta)
     resp = self.session.put(url, data=data, headers=headers)
     if resp.status_code == 201:
         return extract_headers_meta(resp.headers)
     else:
         raise exc.from_response(resp)
예제 #5
0
파일: client.py 프로젝트: hejin/oio-sds
 def chunk_put(self, url, meta, data, **kwargs):
     headers = gen_put_headers(meta)
     resp = self.session.put(url, data=data, headers=headers)
     if resp.status_code == 201:
         return extract_headers_meta(resp.headers)
     else:
         raise exc.from_response(resp)
예제 #6
0
    def chunk_head(self, url, **kwargs):
        """
        Perform a HEAD request on a chunk.

        :param url: URL of the chunk to request.
        :keyword xattr: when False, ask the rawx not to read
            extended attributes of the chunk.
        :keyword check_hash: when True, ask the rawx to validate
            checksum of the chunk.
        :returns: a `dict` with chunk metadata (empty when xattr is False).
        """
        _xattr = bool(kwargs.get('xattr', True))
        headers = kwargs.pop('headers', None)
        # Actually this is not needed since ensure_request_id always sets it
        if headers is None:
            headers = dict()
        else:
            headers = headers.copy()
        headers[FETCHXATTR_HEADER] = _xattr
        if bool(kwargs.get('check_hash', False)):
            headers[CHECKHASH_HEADER] = True

        try:
            resp = self._request('HEAD', url, headers=headers, **kwargs)
        except urllib3.exceptions.HTTPError as ex:
            oio_exception_from_httperror(ex,
                                         reqid=headers[REQID_HEADER],
                                         url=url)
        if resp.status == 200:
            if not _xattr:
                return dict()
            return extract_headers_meta(resp.headers)
        else:
            raise exc.from_response(resp)
예제 #7
0
    def container_create_many(self, account, containers, properties=None,
                              **kwargs):
        """
        Create several containers.

        :param account: account in which to create the containers
        :type account: `str`
        :param containers: names of the containers
        :type containers: iterable of `str`
        :param properties: properties to set on the containers
        :type properties: `dict`
        :keyword headers: extra headers to send to the proxy
        :type headers: `dict`
        :returns: a list of tuples with the name of the container and
            a boolean telling if the container has been created
        :rtype: `list` of `tuple`
        """
        results = list()
        try:
            params = self._make_params(account)
            unformatted_data = list()
            for container in containers:
                unformatted_data.append({'name': container,
                                         'properties': properties or {},
                                         'system': kwargs.get('system', {})})
            data = json.dumps({"containers": unformatted_data})
            resp, body = self._request('POST', '/create_many', params=params,
                                       data=data, autocreate=True,
                                       **kwargs)
            if resp.status not in (204, 200):
                raise exceptions.from_response(resp, body)
            for container in body["containers"]:
                results.append((container["name"], container["status"] == 201))
            return results
        except exceptions.TooLarge:
            # Batch too large for the proxy
            pivot = len(containers) / 2
            head = containers[:pivot]
            tail = containers[pivot:]
            if head:
                results += self.container_create_many(
                        account, head, properties=properties,
                        **kwargs)
            if tail:
                results += self.container_create_many(
                        account, tail, properties=properties,
                        **kwargs)
            return results
        except exceptions.NotFound:
            # Batches not supported by the proxy
            for container in containers:
                try:
                    rc = self.container_create(
                            account, container, properties=properties,
                            **kwargs)
                    results.append((container, rc))
                except Exception:
                    results.append((container, False))
            return results
예제 #8
0
 def chunk_get(self, url, **kwargs):
     resp = self.session.get(url, stream=True)
     if resp.status_code == 200:
         meta = extract_headers_meta(resp.headers)
         stream = resp.iter_content(READ_BUFFER_SIZE)
         return meta, stream
     else:
         raise exc.from_response(resp)
예제 #9
0
파일: client.py 프로젝트: hejin/oio-sds
 def chunk_get(self, url, **kwargs):
     resp = self.session.get(url, stream=True)
     if resp.status_code == 200:
         meta = extract_headers_meta(resp.headers)
         stream = resp.iter_content(READ_BUFFER_SIZE)
         return meta, stream
     else:
         raise exc.from_response(resp)
예제 #10
0
 def test_from_response(self):
     fake_resp = FakeAPIResponse()
     fake_resp.status_code = 500
     exc = exceptions.from_response(fake_resp, None)
     self.assertTrue(isinstance(exc, exceptions.ClientException))
     self.assertEqual(exc.http_status, fake_resp.status_code)
     self.assertEqual(exc.message, "n/a")
     self.assertTrue("HTTP 500" in str(exc))
예제 #11
0
 def test_from_response(self):
     fake_resp = FakeApiResponse()
     fake_resp.status = 500
     exc = exceptions.from_response(fake_resp, None)
     self.assertIsInstance(exc, exceptions.ClientException)
     self.assertEqual(exc.http_status, fake_resp.status)
     self.assertEqual(exc.message, "n/a")
     self.assertIn("HTTP 500", str(exc))
예제 #12
0
 def create(self, account, reference, headers=None):
     uri = self._make_uri('reference/create')
     params = self._make_params(account, reference)
     resp, resp_body = self._request(
         'POST', uri, params=params, headers=headers)
     if resp.status_code in (200, 201):
         return resp_body
     else:
         raise exceptions.from_response(resp, resp_body)
예제 #13
0
파일: client.py 프로젝트: mtanlee/oio-sds
 def _direct_request(self, method, full_url, **kwargs):
     resp = self.session.request(method, full_url, **kwargs)
     try:
         body = resp.json()
     except ValueError:
         body = resp.content
     if resp.status_code >= 400:
         raise from_response(resp, body)
     return resp, body
예제 #14
0
 def chunk_head(self, url, **kwargs):
     _xattr = bool(kwargs.get('xattr', True))
     resp = self.http_pool.request(
         'HEAD', url, headers={HEADER_PREFIX + 'xattr': _xattr})
     if resp.status == 200:
         if not _xattr:
             return dict()
         return extract_headers_meta(resp.headers)
     else:
         raise exc.from_response(resp)
예제 #15
0
 def test_from_response_with_body(self):
     fake_resp = FakeAPIResponse()
     fake_resp.status_code = 500
     body = {"status": 300, "message": "Fake error"}
     exc = exceptions.from_response(fake_resp, body)
     self.assertTrue(isinstance(exc, exceptions.ClientException))
     self.assertEqual(exc.http_status, fake_resp.status_code)
     self.assertEqual(exc.status, 300)
     self.assertEqual(exc.message, "Fake error")
     self.assertTrue("HTTP 500" in str(exc))
     self.assertTrue("STATUS 300" in str(exc))
예제 #16
0
 def _request(self, data):
     try:
         resp = self.http.request(
             "POST",
             self.endpoint,
             headers={'Content-Type': 'application/json'},
             body=data)
         if resp.status >= 400:
             raise exceptions.from_response(resp)
     except urllib3.exceptions.HTTPError as exc:
         oio_exception_from_httperror(exc)
예제 #17
0
 def create(self, account, reference, headers=None):
     uri = self._make_uri('reference/create')
     params = self._make_params(account, reference)
     resp, resp_body = self._request('POST',
                                     uri,
                                     params=params,
                                     headers=headers)
     if resp.status_code in (200, 201):
         return resp_body
     else:
         raise exceptions.from_response(resp, resp_body)
예제 #18
0
파일: client.py 프로젝트: Narthorn/oio-sds
 def _request(self, method, url, **kwargs):
     endpoint = self.endpoint
     url = "/".join([endpoint.rstrip("/"), url.lstrip("/")])
     resp = self.session.request(method, url, **kwargs)
     try:
         body = resp.json()
     except ValueError:
         body = resp.content
     if resp.status_code >= 400:
         raise from_response(resp, body)
     return resp, body
예제 #19
0
 def _direct_request(self, method, full_url, **kwargs):
     headers = kwargs.get('headers') or {}
     headers = dict([k, str(headers[k])] for k in headers)
     kwargs['headers'] = headers
     resp = self.session.request(method, full_url, **kwargs)
     try:
         body = resp.json()
     except ValueError:
         body = resp.content
     if resp.status_code >= 400:
         raise from_response(resp, body)
     return resp, body
예제 #20
0
 def create(self, account, reference, metadata=None, headers=None):
     uri = self._make_uri('reference/create')
     params = self._make_params(account, reference)
     metadata = metadata or {}
     data = json.dumps({'properties': metadata})
     resp, resp_body = self._request(
         'POST', uri, params=params, data=data, headers=headers)
     if resp.status_code not in (201, 202):
         raise exceptions.from_response(resp, resp_body)
     if resp.status_code == 201:
         return True
     else:
         return False
예제 #21
0
 def chunk_head(self, url, **kwargs):
     _xattr = bool(kwargs.get('xattr', True))
     headers = kwargs['headers'].copy()
     headers[HEADER_PREFIX + 'xattr'] = _xattr
     try:
         resp = self.http_pool.request('HEAD', url, headers=headers)
     except urllib3.exceptions.HTTPError as ex:
         oio_exception_from_httperror(ex, headers['X-oio-req-id'])
     if resp.status == 200:
         if not _xattr:
             return dict()
         return extract_headers_meta(resp.headers)
     else:
         raise exc.from_response(resp)
예제 #22
0
파일: base.py 프로젝트: WuYaoWang/oio-sds
    def _direct_request(self, method, url, session=None, admin_mode=False,
                        **kwargs):
        """
        Make an HTTP request.

        :param method: HTTP method to use (e.g. "GET")
        :type method: `str`
        :param url: URL to request
        :type url: `str`
        :param session: the session to use instead of `self.session`
        :type session: requests.Session
        :keyword admin_mode: allow operations on slave or worm namespaces
        :type admin_mode: `bool`
        :keyword timeout: optional timeout for the request (in seconds).
            May be a tuple `(connection_timeout, read_timeout)`.
            This method also accepts `connection_timeout` and `read_timeout`
            as separate arguments.
        :type timeout: `float`
        :keyword headers: optional headers to add to the request
        :type headers: `dict`
        """
        if not session:
            session = self.session

        # Filter arguments that are not recognized by Requests
        out_kwargs = {k: v for k, v in kwargs.items()
                      if k in REQUESTS_KWARGS}

        # Ensure headers are all strings
        headers = kwargs.get('headers') or dict()
        out_headers = {k: str(v) for k, v in headers.items()}
        if self.admin_mode or admin_mode:
            out_headers[ADMIN_HEADER] = '1'
        out_kwargs['headers'] = out_headers

        # Ensure there is a timeout
        if 'timeout' not in out_kwargs:
            out_kwargs['timeout'] = (
                kwargs.get('connection_timeout', CONNECTION_TIMEOUT),
                kwargs.get('read_timeout', READ_TIMEOUT))

        resp = session.request(method, url, **out_kwargs)
        try:
            body = resp.json()
        except ValueError:
            body = resp.content
        if resp.status_code >= 400:
            raise exceptions.from_response(resp, body)
        return resp, body
예제 #23
0
    def container_create(self, account, container, metadata=None, headers=None):
        uri = self._make_uri("container/create")
        params = self._make_params(account, container)

        headers = headers or {}
        headers["x-oio-action-mode"] = "autocreate"
        metadata = metadata or {}
        data = {"properties": metadata}
        resp, resp_body = self._request("POST", uri, params=params, data=json.dumps(data), headers=headers)
        if resp.status_code not in (204, 201):
            raise exc.from_response(resp, resp_body)
        if resp.status_code == 201:
            return False
        else:
            return True
예제 #24
0
파일: base.py 프로젝트: carriercomm/oio-sds
    def _request(self, method, url, endpoint=None, session=None, **kwargs):
        if not endpoint:
            endpoint = self.endpoint
        url = '/'.join([endpoint.rstrip('/'), url.lstrip('/')])
        if not session:
            session = self.session

        resp = session.request(method, url, **kwargs)
        try:
            body = resp.json()
        except ValueError:
            body = resp.content
        if resp.status_code >= 400:
            raise exceptions.from_response(resp, body)
        return resp, body
예제 #25
0
파일: base.py 프로젝트: lanweichang/oio-sds
    def _request(self, method, url, endpoint=None, session=None, **kwargs):
        if not endpoint:
            endpoint = self.endpoint
        url = '/'.join([endpoint.rstrip('/'), url.lstrip('/')])
        if not session:
            session = self.session

        resp = session.request(method, url, **kwargs)
        try:
            body = resp.json()
        except ValueError:
            body = resp.content
        if resp.status_code >= 400:
            raise exceptions.from_response(resp, body)
        return resp, body
예제 #26
0
 def _request(self, method, url, endpoint=None, session=None, **kwargs):
     if not endpoint:
         endpoint = self.endpoint
     url = '/'.join([endpoint.rstrip('/'), url.lstrip('/')])
     if not session:
         session = self.session
     headers = kwargs.get('headers') or {}
     headers = dict([k, str(headers[k])] for k in headers)
     value_admin = "1" if self.admin_mode else "0"
     headers.update({ADMIN_HEADER: value_admin})
     kwargs['headers'] = headers
     resp = session.request(method, url, **kwargs)
     try:
         body = resp.json()
     except ValueError:
         body = resp.content
     if resp.status_code >= 400:
         raise exceptions.from_response(resp, body)
     return resp, body
예제 #27
0
    def container_create(self, account, container, metadata=None,
                         headers=None):
        uri = self._make_uri('container/create')
        params = self._make_params(account, container)

        headers = headers or {}
        headers['x-oio-action-mode'] = 'autocreate'
        if metadata:
            headers_meta = {}
            for k, v in metadata.iteritems():
                headers_meta['%suser-%s' % (
                    constants.CONTAINER_METADATA_PREFIX, k)] = v
            headers.update(headers_meta)
        resp, resp_body = self._request(
            'POST', uri, params=params, headers=headers)
        if resp.status_code not in (204, 201):
            raise exc.from_response(resp, resp_body)
        if resp.status_code == 201:
            return False
        else:
            return True
예제 #28
0
    def container_create(self, account, container, metadata=None,
                         headers=None):
        uri = self._make_uri('container/create')
        params = self._make_params(account, container)

        headers = headers or {}
        headers['x-oio-action-mode'] = 'autocreate'
        if metadata:
            headers_meta = {}
            for k, v in metadata.iteritems():
                headers_meta['%suser-%s' % (
                    constants.CONTAINER_METADATA_PREFIX, k)] = v
            headers.update(headers_meta)
        resp, resp_body = self._request(
            'POST', uri, params=params, headers=headers)
        if resp.status_code not in (204, 201):
            raise exc.from_response(resp, resp_body)
        if resp.status_code == 201:
            return False
        else:
            return True
예제 #29
0
    def create(self, account=None, reference=None, properties=None, **kwargs):
        """
        Create a reference (a service container).

        :param account: name of the account where the reference must be created
        :param reference: name of the reference to create
        :param properties: dictionary of properties to set on the reference
        :type properties: `dict`
        :returns: True if the reference has been created,
            False if it already existed
        """
        params = self._make_params(account, reference)
        if not properties:
            properties = dict()
        data = json.dumps({'properties': properties})
        resp, body = self._request('POST',
                                   '/create',
                                   params=params,
                                   data=data,
                                   **kwargs)
        if resp.status not in (201, 202):
            raise exceptions.from_response(resp, body)
        return resp.status == 201
예제 #30
0
    def _direct_request(self, method, url, session=None, **kwargs):
        """
        Make an HTTP request.

        :param method: HTTP method to use (e.g. "GET")
        :type method: `str`
        :param url: URL to request
        :type url: `str`
        :param session: the session to use instead of `self.session`
        :type session: requests.Session
        :keyword timeout: optional timeout for the request (in seconds).
            May be a tuple `(connection_timeout, response_timeout)`.
        :type timeout: `float`
        :keyword headers: optional headers to add to the request
        :type headers: `dict`
        """
        if not session:
            session = self.session
        in_headers = kwargs.get('headers') or dict()
        headers = {k: str(v) for k, v in in_headers.items()}
        if self.admin_mode or kwargs.get('admin_mode', False):
            headers.update({ADMIN_HEADER: "1"})
        kwargs['headers'] = headers
        if "timeout" not in kwargs:
            resp = session.request(method,
                                   url,
                                   timeout=(CONNECTION_TIMEOUT, READ_TIMEOUT),
                                   **kwargs)
        else:
            resp = session.request(method, url, **kwargs)
        try:
            body = resp.json()
        except ValueError:
            body = resp.content
        if resp.status_code >= 400:
            raise exceptions.from_response(resp, body)
        return resp, body
예제 #31
0
    def chunk_head(self, url, **kwargs):
        """
        Perform a HEAD request on a chunk.

        :param url: URL of the chunk to request.
        :keyword xattr: when False, ask the rawx not to read
            extended attributes of the chunk.
        :keyword check_hash: when True, ask the rawx to validate
            checksum of the chunk.
        :returns: a `dict` with chunk metadata (empty when xattr is False).
        """
        _xattr = bool(kwargs.get('xattr', True))
        url = self.resolve_url(url)
        headers = kwargs['headers'].copy()
        headers[FETCHXATTR_HEADER] = _xattr
        if bool(kwargs.get('check_hash', False)):
            headers[CHECKHASH_HEADER] = True
        timeout = kwargs.get('timeout')
        if not timeout:
            timeout = urllib3.Timeout(CHUNK_TIMEOUT)

        try:
            resp = self.http_pool.request('HEAD',
                                          url,
                                          headers=headers,
                                          timeout=timeout)
        except urllib3.exceptions.HTTPError as ex:
            oio_exception_from_httperror(ex,
                                         reqid=headers[REQID_HEADER],
                                         url=url)
        if resp.status == 200:
            if not _xattr:
                return dict()
            return extract_headers_meta(resp.headers)
        else:
            raise exc.from_response(resp)
예제 #32
0
 def chunk_delete(self, url, **kwargs):
     resp = self.session.delete(url)
     if resp.status_code != 204:
         raise exc.from_response(resp)
예제 #33
0
 def chunk_delete(self, url, **kwargs):
     resp = self._request('DELETE', url, **kwargs)
     if resp.status != 204:
         raise exc.from_response(resp)
     return resp
예제 #34
0
    def _direct_request(self, method, url, headers=None, data=None, json=None,
                        params=None, admin_mode=False, pool_manager=None,
                        force_master=False, **kwargs):
        """
        Make an HTTP request.

        :param method: HTTP method to use (e.g. "GET")
        :type method: `str`
        :param url: URL to request
        :type url: `str`
        :keyword admin_mode: allow operations on slave or worm namespaces
        :type admin_mode: `bool`
        :keyword deadline: deadline for the request, in monotonic time.
            Supersedes `read_timeout`.
        :type deadline: `float` seconds
        :keyword timeout: optional timeout for the request (in seconds).
            May be a `urllib3.Timeout(connect=connection_timeout,
            read=read_timeout)`.
            This method also accepts `connection_timeout` and `read_timeout`
            as separate arguments.
        :type timeout: `float` or `urllib3.Timeout`
        :keyword headers: optional headers to add to the request
        :type headers: `dict`
        :keyword force_master: request will run on master service only.
        :type force_master: `bool`

        :raise oio.common.exceptions.OioTimeout: in case of read, write
        or connection timeout
        :raise oio.common.exceptions.OioNetworkException: in case of
        connection error
        :raise oio.common.exceptions.OioException: in other case of HTTP error
        :raise oio.common.exceptions.ClientException: in case of HTTP status
        code >= 400
        """
        out_kwargs = dict(kwargs)

        # Ensure headers are all strings
        if headers:
            out_headers = {k: str(v) for k, v in headers.items()}
        else:
            out_headers = dict()
        if self.admin_mode or admin_mode:
            out_headers[ADMIN_HEADER] = '1'
        if self.force_master or force_master:
            out_headers[FORCEMASTER_HEADER] = '1'

        # Look for a request deadline, deduce the timeout from it.
        if kwargs.get('deadline', None) is not None:
            to = deadline_to_timeout(kwargs['deadline'], True)
            to = min(to, kwargs.get('read_timeout', to))
            out_kwargs['timeout'] = urllib3.Timeout(
                connect=kwargs.get('connection_timeout', CONNECTION_TIMEOUT),
                read=to)
            # Shorten the deadline by 1% to compensate for the time spent
            # connecting and reading response.
            out_headers[TIMEOUT_HEADER] = int(to * 990000.0)

        # Ensure there is a timeout
        if 'timeout' not in out_kwargs:
            out_kwargs['timeout'] = urllib3.Timeout(
                connect=kwargs.get('connection_timeout', CONNECTION_TIMEOUT),
                read=kwargs.get('read_timeout', READ_TIMEOUT))
        if TIMEOUT_HEADER not in out_headers:
            to = out_kwargs['timeout']
            if isinstance(to, urllib3.Timeout):
                to = to.read_timeout
            else:
                to = float(to)
            out_headers[TIMEOUT_HEADER] = int(to * 1000000.0)

        # Look for a request ID
        if 'reqid' in kwargs:
            out_headers[REQID_HEADER] = str(kwargs['reqid'])

        if len(out_headers.get(REQID_HEADER, '')) > STRLEN_REQID:
            out_headers[REQID_HEADER] = \
                out_headers[REQID_HEADER][:STRLEN_REQID]
            self.__logger().warn('Request ID truncated to %d characters',
                                 STRLEN_REQID)

        # Convert json and add Content-Type
        if json:
            out_headers["Content-Type"] = "application/json"
            data = jsonlib.dumps(json)

        # Trigger performance measurments
        perfdata = kwargs.get('perfdata', None)
        if perfdata is not None:
            out_headers[PERFDATA_HEADER] = 'enabled'

        # Explicitly keep or close the connection
        if 'Connection' not in out_headers:
            out_headers['Connection'] = self.connection

        out_kwargs['headers'] = out_headers
        out_kwargs['body'] = data

        # Add query string
        if params:
            out_param = []
            for k, v in params.items():
                if v is not None:
                    if isinstance(v, unicode):
                        v = unicode(v).encode('utf-8')
                    out_param.append((k, v))
            encoded_args = urlencode(out_param)
            url += '?' + encoded_args

        if not pool_manager:
            pool_manager = self.pool_manager

        try:
            if perfdata is not None:
                request_start = monotonic_time()
            resp = pool_manager.request(method, url, **out_kwargs)
            if perfdata is not None:
                request_end = monotonic_time()
                service_perfdata = perfdata.setdefault(
                    self.service_type, dict())
                service_perfdata['total'] = service_perfdata.get(
                    'total', 0.0) + request_end - request_start
            body = resp.data
            if body:
                try:
                    body = jsonlib.loads(body)
                except ValueError:
                    pass
            if perfdata is not None and PERFDATA_HEADER in resp.headers:
                service_perfdata = perfdata[self.service_type]
                for header_val in resp.headers[PERFDATA_HEADER].split(','):
                    kv = header_val.split('=', 1)
                    service_perfdata[kv[0]] = service_perfdata.get(
                        kv[0], 0.0) + float(kv[1]) / 1000000.0
        except urllib3.exceptions.HTTPError as exc:
            oio_exception_from_httperror(exc,
                                         reqid=out_headers.get(REQID_HEADER),
                                         url=url)
        if resp.status >= 400:
            raise exceptions.from_response(resp, body)
        return resp, body
예제 #35
0
 def chunk_head(self, url, **kwargs):
     resp = self.session.head(url)
     if resp.status_code == 200:
         return extract_headers_meta(resp.headers)
     else:
         raise exc.from_response(resp)
예제 #36
0
파일: base.py 프로젝트: vdombrovski/oio-sds
    def _direct_request(self,
                        method,
                        url,
                        headers=None,
                        data=None,
                        json=None,
                        params=None,
                        admin_mode=False,
                        pool_manager=None,
                        **kwargs):
        """
        Make an HTTP request.

        :param method: HTTP method to use (e.g. "GET")
        :type method: `str`
        :param url: URL to request
        :type url: `str`
        :keyword admin_mode: allow operations on slave or worm namespaces
        :type admin_mode: `bool`
        :keyword deadline: deadline for the request, in monotonic time.
            Supersedes `read_timeout`.
        :type deadline: `float` seconds
        :keyword timeout: optional timeout for the request (in seconds).
            May be a `urllib3.Timeout(connect=connection_timeout,
            read=read_timeout)`.
            This method also accepts `connection_timeout` and `read_timeout`
            as separate arguments.
        :type timeout: `float` or `urllib3.Timeout`
        :keyword headers: optional headers to add to the request
        :type headers: `dict`

        :raise oio.common.exceptions.OioTimeout: in case of read, write
        or connection timeout
        :raise oio.common.exceptions.OioNetworkException: in case of
        connection error
        :raise oio.common.exceptions.OioException: in other case of HTTP error
        :raise oio.common.exceptions.ClientException: in case of HTTP status
        code >= 400
        """
        # Filter arguments that are not recognized by Requests
        out_kwargs = {
            k: v
            for k, v in iteritems(kwargs) if k in URLLIB3_REQUESTS_KWARGS
        }

        # Ensure headers are all strings
        if headers:
            out_headers = {k: text_type(v) for k, v in headers.items()}
        else:
            out_headers = dict()
        if self.admin_mode or admin_mode:
            out_headers[ADMIN_HEADER] = '1'

        # Look for a request deadline, deduce the timeout from it.
        if kwargs.get('deadline', None) is not None:
            to = deadline_to_timeout(kwargs['deadline'], True)
            to = min(to, kwargs.get('read_timeout', to))
            out_kwargs['timeout'] = urllib3.Timeout(connect=kwargs.get(
                'connection_timeout', CONNECTION_TIMEOUT),
                                                    read=to)
            # Shorten the deadline by 1% to compensate for the time spent
            # connecting and reading response.
            out_headers[TIMEOUT_HEADER] = int(to * 990000.0)

        # Ensure there is a timeout
        if 'timeout' not in out_kwargs:
            out_kwargs['timeout'] = urllib3.Timeout(
                connect=kwargs.get('connection_timeout', CONNECTION_TIMEOUT),
                read=kwargs.get('read_timeout', READ_TIMEOUT))
        if TIMEOUT_HEADER not in out_headers:
            to = out_kwargs['timeout']
            if isinstance(to, urllib3.Timeout):
                to = to.read_timeout
            else:
                to = float(to)
            out_headers[TIMEOUT_HEADER] = int(to * 1000000.0)

        # Convert json and add Content-Type
        if json:
            out_headers["Content-Type"] = "application/json"
            data = jsonlib.dumps(json)

        # Trigger performance measurments
        perfdata = kwargs.get('perfdata', self.perfdata)
        if perfdata is not None:
            out_headers[PERFDATA_HEADER] = 'enabled'

        out_kwargs['headers'] = out_headers
        out_kwargs['body'] = data

        # Add query string
        if params:
            out_param = []
            for k, v in params.items():
                if v is not None:
                    if isinstance(v, text_type):
                        v = text_type(v).encode('utf-8')
                    out_param.append((k, v))
            encoded_args = urlencode(out_param)
            url += '?' + encoded_args

        if not pool_manager:
            pool_manager = self.pool_manager

        def _reraise(exc_type, exc_value):
            reqid = out_headers.get('X-oio-req-id')
            exceptions.reraise(exc_type, exc_value, "reqid=%s" % reqid)

        try:
            resp = pool_manager.request(method, url, **out_kwargs)
            body = resp.data
            if body:
                try:
                    body = jsonlib.loads(body)
                except ValueError:
                    pass
            if perfdata is not None and PERFDATA_HEADER in resp.headers:
                for header_val in resp.headers[PERFDATA_HEADER].split(','):
                    kv = header_val.split('=', 1)
                    pdat = perfdata.get(kv[0], 0.0) + float(kv[1]) / 1000000.0
                    perfdata[kv[0]] = pdat
        except MaxRetryError as exc:
            if isinstance(exc.reason, NewConnectionError):
                _reraise(exceptions.OioNetworkException, exc)
            if isinstance(exc.reason, TimeoutError):
                _reraise(exceptions.OioTimeout, exc)
            _reraise(exceptions.OioNetworkException, exc)
        except (ProtocolError, ProxyError, ClosedPoolError) as exc:
            _reraise(exceptions.OioNetworkException, exc)
        except TimeoutError as exc:
            _reraise(exceptions.OioTimeout, exc)
        except HTTPError as exc:
            _reraise(exceptions.OioException, exc)

        if resp.status >= 400:
            raise exceptions.from_response(resp, body)
        return resp, body
예제 #37
0
 def chunk_delete(self, url, **kwargs):
     resp = self.session.delete(url)
     if resp.status_code != 204:
         raise exc.from_response(resp)
예제 #38
0
파일: base.py 프로젝트: kamel-rahim/oio-sds
    def _direct_request(self,
                        method,
                        url,
                        headers=None,
                        data=None,
                        json=None,
                        params=None,
                        admin_mode=False,
                        pool_manager=None,
                        **kwargs):
        """
        Make an HTTP request.

        :param method: HTTP method to use (e.g. "GET")
        :type method: `str`
        :param url: URL to request
        :type url: `str`
        :keyword admin_mode: allow operations on slave or worm namespaces
        :type admin_mode: `bool`
        :keyword timeout: optional timeout for the request (in seconds).
            May be a `urllib3.Timeout(connect=connection_timeout,
            read=read_timeout)`.
            This method also accepts `connection_timeout` and `read_timeout`
            as separate arguments.
        :type timeout: `float` or `urllib3.Timeout`
        :keyword headers: optional headers to add to the request
        :type headers: `dict`

        :raise oio.common.exceptions.OioTimeout: in case of read, write
        or connection timeout
        :raise oio.common.exceptions.OioNetworkException: in case of
        connection error
        :raise oio.common.exceptions.OioException: in other case of HTTP error
        :raise oio.common.exceptions.ClientException: in case of HTTP status
        code >= 400
        """
        # Filter arguments that are not recognized by Requests
        out_kwargs = {
            k: v
            for k, v in kwargs.items() if k in URLLIB3_REQUESTS_KWARGS
        }

        # Ensure headers are all strings
        if headers:
            out_headers = {k: str(v) for k, v in headers.items()}
        else:
            out_headers = dict()
        if self.admin_mode or admin_mode:
            out_headers[ADMIN_HEADER] = '1'

        # Ensure there is a timeout
        if 'timeout' not in out_kwargs:
            out_kwargs['timeout'] = urllib3.Timeout(
                connect=kwargs.get('connection_timeout', CONNECTION_TIMEOUT),
                read=kwargs.get('read_timeout', READ_TIMEOUT))

        # Convert json and add Content-Type
        if json:
            out_headers["Content-Type"] = "application/json"
            data = jsonlib.dumps(json)

        out_kwargs['headers'] = out_headers
        out_kwargs['body'] = data

        # Add query string
        if params:
            out_param = []
            for k, v in params.items():
                if v is not None:
                    if isinstance(v, unicode):
                        v = unicode(v).encode('utf-8')
                    out_param.append((k, v))
            encoded_args = urlencode(out_param)
            url += '?' + encoded_args

        if not pool_manager:
            pool_manager = self.pool_manager

        try:
            resp = pool_manager.request(method, url, **out_kwargs)
            body = resp.data
            if body:
                try:
                    body = jsonlib.loads(body)
                except ValueError:
                    pass
        except MaxRetryError as exc:
            if isinstance(exc.reason, NewConnectionError):
                raise exceptions.OioNetworkException(exc), None, \
                        sys.exc_info()[2]
            if isinstance(exc.reason, TimeoutError):
                raise exceptions.OioTimeout(exc), None, sys.exc_info()[2]
            raise exceptions.OioNetworkException(exc), None, sys.exc_info()[2]
        except (ProtocolError, ProxyError, ClosedPoolError) as exc:
            raise exceptions.OioNetworkException(exc), None, sys.exc_info()[2]
        except TimeoutError as exc:
            raise exceptions.OioTimeout(exc), None, sys.exc_info()[2]
        except HTTPError as exc:
            raise exceptions.OioException(exc), None, sys.exc_info()[2]
        if resp.status >= 400:
            raise exceptions.from_response(resp, body)
        return resp, body
예제 #39
0
 def chunk_delete(self, url, **kwargs):
     resp = self.http_pool.request('DELETE', self.resolve_url(url),
                                   **kwargs)
     if resp.status != 204:
         raise exc.from_response(resp)
     return resp
예제 #40
0
 def chunk_head(self, url, **kwargs):
     resp = self.session.head(url)
     if resp.status_code == 200:
         return extract_headers_meta(resp.headers)
     else:
         raise exc.from_response(resp)