def test_from_response_http_status(self):
fake_resp = FakeAPIResponse()
fake_resp.status_code = 404
exc = exceptions.from_response(fake_resp, None)
self.assertTrue(isinstance(exc, exceptions.NotFound))
fake_resp.status_code = 409
exc = exceptions.from_response(fake_resp, None)
self.assertTrue(isinstance(exc, exceptions.Conflict))
def test_from_response_http_status(self):
fake_resp = FakeApiResponse()
fake_resp.status = 404
exc = exceptions.from_response(fake_resp, None)
self.assertIsInstance(exc, exceptions.NotFound)
fake_resp.status = 409
exc = exceptions.from_response(fake_resp, None)
self.assertIsInstance(exc, exceptions.Conflict)
def container_create(self,
account,
reference,
properties=None,
system=None,
**kwargs):
"""
Create a container.
:param account: account in which to create the container
:type account: `str`
:param reference: name of the container
:type reference: `str`
:param properties: properties to set on the container
:type properties: `dict`
:param system: system properties to set on the container
:type system: `dict`
:keyword headers: extra headers to send to the proxy
:type headers: `dict`
:returns: True if the container has been created,
False if it already exists
"""
params = self._make_params(account, reference)
data = json.dumps({
'properties': properties or {},
'system': system or {}
})
resp, body = self._request('POST',
'/create',
params=params,
data=data,
**kwargs)
if resp.status not in (204, 201):
raise exceptions.from_response(resp, body)
return resp.status == 201
def chunk_put(self, url, meta, data, **kwargs):
headers = gen_put_headers(meta)
resp = self.session.put(url, data=data, headers=headers)
if resp.status_code == 201:
return extract_headers_meta(resp.headers)
else:
raise exc.from_response(resp)
def chunk_put(self, url, meta, data, **kwargs):
headers = gen_put_headers(meta)
resp = self.session.put(url, data=data, headers=headers)
if resp.status_code == 201:
return extract_headers_meta(resp.headers)
else:
raise exc.from_response(resp)
def chunk_head(self, url, **kwargs):
"""
Perform a HEAD request on a chunk.
:param url: URL of the chunk to request.
:keyword xattr: when False, ask the rawx not to read
extended attributes of the chunk.
:keyword check_hash: when True, ask the rawx to validate
checksum of the chunk.
:returns: a `dict` with chunk metadata (empty when xattr is False).
"""
_xattr = bool(kwargs.get('xattr', True))
headers = kwargs.pop('headers', None)
# Actually this is not needed since ensure_request_id always sets it
if headers is None:
headers = dict()
else:
headers = headers.copy()
headers[FETCHXATTR_HEADER] = _xattr
if bool(kwargs.get('check_hash', False)):
headers[CHECKHASH_HEADER] = True
try:
resp = self._request('HEAD', url, headers=headers, **kwargs)
except urllib3.exceptions.HTTPError as ex:
oio_exception_from_httperror(ex,
reqid=headers[REQID_HEADER],
url=url)
if resp.status == 200:
if not _xattr:
return dict()
return extract_headers_meta(resp.headers)
else:
raise exc.from_response(resp)
def container_create_many(self, account, containers, properties=None,
**kwargs):
"""
Create several containers.
:param account: account in which to create the containers
:type account: `str`
:param containers: names of the containers
:type containers: iterable of `str`
:param properties: properties to set on the containers
:type properties: `dict`
:keyword headers: extra headers to send to the proxy
:type headers: `dict`
:returns: a list of tuples with the name of the container and
a boolean telling if the container has been created
:rtype: `list` of `tuple`
"""
results = list()
try:
params = self._make_params(account)
unformatted_data = list()
for container in containers:
unformatted_data.append({'name': container,
'properties': properties or {},
'system': kwargs.get('system', {})})
data = json.dumps({"containers": unformatted_data})
resp, body = self._request('POST', '/create_many', params=params,
data=data, autocreate=True,
**kwargs)
if resp.status not in (204, 200):
raise exceptions.from_response(resp, body)
for container in body["containers"]:
results.append((container["name"], container["status"] == 201))
return results
except exceptions.TooLarge:
# Batch too large for the proxy
pivot = len(containers) / 2
head = containers[:pivot]
tail = containers[pivot:]
if head:
results += self.container_create_many(
account, head, properties=properties,
**kwargs)
if tail:
results += self.container_create_many(
account, tail, properties=properties,
**kwargs)
return results
except exceptions.NotFound:
# Batches not supported by the proxy
for container in containers:
try:
rc = self.container_create(
account, container, properties=properties,
**kwargs)
results.append((container, rc))
except Exception:
results.append((container, False))
return results
def chunk_get(self, url, **kwargs):
resp = self.session.get(url, stream=True)
if resp.status_code == 200:
meta = extract_headers_meta(resp.headers)
stream = resp.iter_content(READ_BUFFER_SIZE)
return meta, stream
else:
raise exc.from_response(resp)
def chunk_get(self, url, **kwargs):
resp = self.session.get(url, stream=True)
if resp.status_code == 200:
meta = extract_headers_meta(resp.headers)
stream = resp.iter_content(READ_BUFFER_SIZE)
return meta, stream
else:
raise exc.from_response(resp)
def test_from_response(self):
fake_resp = FakeAPIResponse()
fake_resp.status_code = 500
exc = exceptions.from_response(fake_resp, None)
self.assertTrue(isinstance(exc, exceptions.ClientException))
self.assertEqual(exc.http_status, fake_resp.status_code)
self.assertEqual(exc.message, "n/a")
self.assertTrue("HTTP 500" in str(exc))
def test_from_response(self):
fake_resp = FakeApiResponse()
fake_resp.status = 500
exc = exceptions.from_response(fake_resp, None)
self.assertIsInstance(exc, exceptions.ClientException)
self.assertEqual(exc.http_status, fake_resp.status)
self.assertEqual(exc.message, "n/a")
self.assertIn("HTTP 500", str(exc))
def create(self, account, reference, headers=None):
uri = self._make_uri('reference/create')
params = self._make_params(account, reference)
resp, resp_body = self._request(
'POST', uri, params=params, headers=headers)
if resp.status_code in (200, 201):
return resp_body
else:
raise exceptions.from_response(resp, resp_body)
def _direct_request(self, method, full_url, **kwargs):
resp = self.session.request(method, full_url, **kwargs)
try:
body = resp.json()
except ValueError:
body = resp.content
if resp.status_code >= 400:
raise from_response(resp, body)
return resp, body
def chunk_head(self, url, **kwargs):
_xattr = bool(kwargs.get('xattr', True))
resp = self.http_pool.request(
'HEAD', url, headers={HEADER_PREFIX + 'xattr': _xattr})
if resp.status == 200:
if not _xattr:
return dict()
return extract_headers_meta(resp.headers)
else:
raise exc.from_response(resp)
def test_from_response_with_body(self):
fake_resp = FakeAPIResponse()
fake_resp.status_code = 500
body = {"status": 300, "message": "Fake error"}
exc = exceptions.from_response(fake_resp, body)
self.assertTrue(isinstance(exc, exceptions.ClientException))
self.assertEqual(exc.http_status, fake_resp.status_code)
self.assertEqual(exc.status, 300)
self.assertEqual(exc.message, "Fake error")
self.assertTrue("HTTP 500" in str(exc))
self.assertTrue("STATUS 300" in str(exc))
def _request(self, data):
try:
resp = self.http.request(
"POST",
self.endpoint,
headers={'Content-Type': 'application/json'},
body=data)
if resp.status >= 400:
raise exceptions.from_response(resp)
except urllib3.exceptions.HTTPError as exc:
oio_exception_from_httperror(exc)
def create(self, account, reference, headers=None):
uri = self._make_uri('reference/create')
params = self._make_params(account, reference)
resp, resp_body = self._request('POST',
uri,
params=params,
headers=headers)
if resp.status_code in (200, 201):
return resp_body
else:
raise exceptions.from_response(resp, resp_body)
def _request(self, method, url, **kwargs):
endpoint = self.endpoint
url = "/".join([endpoint.rstrip("/"), url.lstrip("/")])
resp = self.session.request(method, url, **kwargs)
try:
body = resp.json()
except ValueError:
body = resp.content
if resp.status_code >= 400:
raise from_response(resp, body)
return resp, body
def _direct_request(self, method, full_url, **kwargs):
headers = kwargs.get('headers') or {}
headers = dict([k, str(headers[k])] for k in headers)
kwargs['headers'] = headers
resp = self.session.request(method, full_url, **kwargs)
try:
body = resp.json()
except ValueError:
body = resp.content
if resp.status_code >= 400:
raise from_response(resp, body)
return resp, body
def create(self, account, reference, metadata=None, headers=None):
uri = self._make_uri('reference/create')
params = self._make_params(account, reference)
metadata = metadata or {}
data = json.dumps({'properties': metadata})
resp, resp_body = self._request(
'POST', uri, params=params, data=data, headers=headers)
if resp.status_code not in (201, 202):
raise exceptions.from_response(resp, resp_body)
if resp.status_code == 201:
return True
else:
return False
def chunk_head(self, url, **kwargs):
_xattr = bool(kwargs.get('xattr', True))
headers = kwargs['headers'].copy()
headers[HEADER_PREFIX + 'xattr'] = _xattr
try:
resp = self.http_pool.request('HEAD', url, headers=headers)
except urllib3.exceptions.HTTPError as ex:
oio_exception_from_httperror(ex, headers['X-oio-req-id'])
if resp.status == 200:
if not _xattr:
return dict()
return extract_headers_meta(resp.headers)
else:
raise exc.from_response(resp)
def _direct_request(self, method, url, session=None, admin_mode=False,
**kwargs):
"""
Make an HTTP request.
:param method: HTTP method to use (e.g. "GET")
:type method: `str`
:param url: URL to request
:type url: `str`
:param session: the session to use instead of `self.session`
:type session: requests.Session
:keyword admin_mode: allow operations on slave or worm namespaces
:type admin_mode: `bool`
:keyword timeout: optional timeout for the request (in seconds).
May be a tuple `(connection_timeout, read_timeout)`.
This method also accepts `connection_timeout` and `read_timeout`
as separate arguments.
:type timeout: `float`
:keyword headers: optional headers to add to the request
:type headers: `dict`
"""
if not session:
session = self.session
# Filter arguments that are not recognized by Requests
out_kwargs = {k: v for k, v in kwargs.items()
if k in REQUESTS_KWARGS}
# Ensure headers are all strings
headers = kwargs.get('headers') or dict()
out_headers = {k: str(v) for k, v in headers.items()}
if self.admin_mode or admin_mode:
out_headers[ADMIN_HEADER] = '1'
out_kwargs['headers'] = out_headers
# Ensure there is a timeout
if 'timeout' not in out_kwargs:
out_kwargs['timeout'] = (
kwargs.get('connection_timeout', CONNECTION_TIMEOUT),
kwargs.get('read_timeout', READ_TIMEOUT))
resp = session.request(method, url, **out_kwargs)
try:
body = resp.json()
except ValueError:
body = resp.content
if resp.status_code >= 400:
raise exceptions.from_response(resp, body)
return resp, body
def container_create(self, account, container, metadata=None, headers=None):
uri = self._make_uri("container/create")
params = self._make_params(account, container)
headers = headers or {}
headers["x-oio-action-mode"] = "autocreate"
metadata = metadata or {}
data = {"properties": metadata}
resp, resp_body = self._request("POST", uri, params=params, data=json.dumps(data), headers=headers)
if resp.status_code not in (204, 201):
raise exc.from_response(resp, resp_body)
if resp.status_code == 201:
return False
else:
return True
def _request(self, method, url, endpoint=None, session=None, **kwargs):
if not endpoint:
endpoint = self.endpoint
url = '/'.join([endpoint.rstrip('/'), url.lstrip('/')])
if not session:
session = self.session
resp = session.request(method, url, **kwargs)
try:
body = resp.json()
except ValueError:
body = resp.content
if resp.status_code >= 400:
raise exceptions.from_response(resp, body)
return resp, body
def _request(self, method, url, endpoint=None, session=None, **kwargs):
if not endpoint:
endpoint = self.endpoint
url = '/'.join([endpoint.rstrip('/'), url.lstrip('/')])
if not session:
session = self.session
resp = session.request(method, url, **kwargs)
try:
body = resp.json()
except ValueError:
body = resp.content
if resp.status_code >= 400:
raise exceptions.from_response(resp, body)
return resp, body
def _request(self, method, url, endpoint=None, session=None, **kwargs):
if not endpoint:
endpoint = self.endpoint
url = '/'.join([endpoint.rstrip('/'), url.lstrip('/')])
if not session:
session = self.session
headers = kwargs.get('headers') or {}
headers = dict([k, str(headers[k])] for k in headers)
value_admin = "1" if self.admin_mode else "0"
headers.update({ADMIN_HEADER: value_admin})
kwargs['headers'] = headers
resp = session.request(method, url, **kwargs)
try:
body = resp.json()
except ValueError:
body = resp.content
if resp.status_code >= 400:
raise exceptions.from_response(resp, body)
return resp, body
def container_create(self, account, container, metadata=None,
headers=None):
uri = self._make_uri('container/create')
params = self._make_params(account, container)
headers = headers or {}
headers['x-oio-action-mode'] = 'autocreate'
if metadata:
headers_meta = {}
for k, v in metadata.iteritems():
headers_meta['%suser-%s' % (
constants.CONTAINER_METADATA_PREFIX, k)] = v
headers.update(headers_meta)
resp, resp_body = self._request(
'POST', uri, params=params, headers=headers)
if resp.status_code not in (204, 201):
raise exc.from_response(resp, resp_body)
if resp.status_code == 201:
return False
else:
return True
def container_create(self, account, container, metadata=None,
headers=None):
uri = self._make_uri('container/create')
params = self._make_params(account, container)
headers = headers or {}
headers['x-oio-action-mode'] = 'autocreate'
if metadata:
headers_meta = {}
for k, v in metadata.iteritems():
headers_meta['%suser-%s' % (
constants.CONTAINER_METADATA_PREFIX, k)] = v
headers.update(headers_meta)
resp, resp_body = self._request(
'POST', uri, params=params, headers=headers)
if resp.status_code not in (204, 201):
raise exc.from_response(resp, resp_body)
if resp.status_code == 201:
return False
else:
return True
def create(self, account=None, reference=None, properties=None, **kwargs):
"""
Create a reference (a service container).
:param account: name of the account where the reference must be created
:param reference: name of the reference to create
:param properties: dictionary of properties to set on the reference
:type properties: `dict`
:returns: True if the reference has been created,
False if it already existed
"""
params = self._make_params(account, reference)
if not properties:
properties = dict()
data = json.dumps({'properties': properties})
resp, body = self._request('POST',
'/create',
params=params,
data=data,
**kwargs)
if resp.status not in (201, 202):
raise exceptions.from_response(resp, body)
return resp.status == 201
def _direct_request(self, method, url, session=None, **kwargs):
"""
Make an HTTP request.
:param method: HTTP method to use (e.g. "GET")
:type method: `str`
:param url: URL to request
:type url: `str`
:param session: the session to use instead of `self.session`
:type session: requests.Session
:keyword timeout: optional timeout for the request (in seconds).
May be a tuple `(connection_timeout, response_timeout)`.
:type timeout: `float`
:keyword headers: optional headers to add to the request
:type headers: `dict`
"""
if not session:
session = self.session
in_headers = kwargs.get('headers') or dict()
headers = {k: str(v) for k, v in in_headers.items()}
if self.admin_mode or kwargs.get('admin_mode', False):
headers.update({ADMIN_HEADER: "1"})
kwargs['headers'] = headers
if "timeout" not in kwargs:
resp = session.request(method,
url,
timeout=(CONNECTION_TIMEOUT, READ_TIMEOUT),
**kwargs)
else:
resp = session.request(method, url, **kwargs)
try:
body = resp.json()
except ValueError:
body = resp.content
if resp.status_code >= 400:
raise exceptions.from_response(resp, body)
return resp, body
def chunk_head(self, url, **kwargs):
"""
Perform a HEAD request on a chunk.
:param url: URL of the chunk to request.
:keyword xattr: when False, ask the rawx not to read
extended attributes of the chunk.
:keyword check_hash: when True, ask the rawx to validate
checksum of the chunk.
:returns: a `dict` with chunk metadata (empty when xattr is False).
"""
_xattr = bool(kwargs.get('xattr', True))
url = self.resolve_url(url)
headers = kwargs['headers'].copy()
headers[FETCHXATTR_HEADER] = _xattr
if bool(kwargs.get('check_hash', False)):
headers[CHECKHASH_HEADER] = True
timeout = kwargs.get('timeout')
if not timeout:
timeout = urllib3.Timeout(CHUNK_TIMEOUT)
try:
resp = self.http_pool.request('HEAD',
url,
headers=headers,
timeout=timeout)
except urllib3.exceptions.HTTPError as ex:
oio_exception_from_httperror(ex,
reqid=headers[REQID_HEADER],
url=url)
if resp.status == 200:
if not _xattr:
return dict()
return extract_headers_meta(resp.headers)
else:
raise exc.from_response(resp)
def chunk_delete(self, url, **kwargs):
resp = self.session.delete(url)
if resp.status_code != 204:
raise exc.from_response(resp)
def chunk_delete(self, url, **kwargs):
resp = self._request('DELETE', url, **kwargs)
if resp.status != 204:
raise exc.from_response(resp)
return resp
def _direct_request(self, method, url, headers=None, data=None, json=None,
params=None, admin_mode=False, pool_manager=None,
force_master=False, **kwargs):
"""
Make an HTTP request.
:param method: HTTP method to use (e.g. "GET")
:type method: `str`
:param url: URL to request
:type url: `str`
:keyword admin_mode: allow operations on slave or worm namespaces
:type admin_mode: `bool`
:keyword deadline: deadline for the request, in monotonic time.
Supersedes `read_timeout`.
:type deadline: `float` seconds
:keyword timeout: optional timeout for the request (in seconds).
May be a `urllib3.Timeout(connect=connection_timeout,
read=read_timeout)`.
This method also accepts `connection_timeout` and `read_timeout`
as separate arguments.
:type timeout: `float` or `urllib3.Timeout`
:keyword headers: optional headers to add to the request
:type headers: `dict`
:keyword force_master: request will run on master service only.
:type force_master: `bool`
:raise oio.common.exceptions.OioTimeout: in case of read, write
or connection timeout
:raise oio.common.exceptions.OioNetworkException: in case of
connection error
:raise oio.common.exceptions.OioException: in other case of HTTP error
:raise oio.common.exceptions.ClientException: in case of HTTP status
code >= 400
"""
out_kwargs = dict(kwargs)
# Ensure headers are all strings
if headers:
out_headers = {k: str(v) for k, v in headers.items()}
else:
out_headers = dict()
if self.admin_mode or admin_mode:
out_headers[ADMIN_HEADER] = '1'
if self.force_master or force_master:
out_headers[FORCEMASTER_HEADER] = '1'
# Look for a request deadline, deduce the timeout from it.
if kwargs.get('deadline', None) is not None:
to = deadline_to_timeout(kwargs['deadline'], True)
to = min(to, kwargs.get('read_timeout', to))
out_kwargs['timeout'] = urllib3.Timeout(
connect=kwargs.get('connection_timeout', CONNECTION_TIMEOUT),
read=to)
# Shorten the deadline by 1% to compensate for the time spent
# connecting and reading response.
out_headers[TIMEOUT_HEADER] = int(to * 990000.0)
# Ensure there is a timeout
if 'timeout' not in out_kwargs:
out_kwargs['timeout'] = urllib3.Timeout(
connect=kwargs.get('connection_timeout', CONNECTION_TIMEOUT),
read=kwargs.get('read_timeout', READ_TIMEOUT))
if TIMEOUT_HEADER not in out_headers:
to = out_kwargs['timeout']
if isinstance(to, urllib3.Timeout):
to = to.read_timeout
else:
to = float(to)
out_headers[TIMEOUT_HEADER] = int(to * 1000000.0)
# Look for a request ID
if 'reqid' in kwargs:
out_headers[REQID_HEADER] = str(kwargs['reqid'])
if len(out_headers.get(REQID_HEADER, '')) > STRLEN_REQID:
out_headers[REQID_HEADER] = \
out_headers[REQID_HEADER][:STRLEN_REQID]
self.__logger().warn('Request ID truncated to %d characters',
STRLEN_REQID)
# Convert json and add Content-Type
if json:
out_headers["Content-Type"] = "application/json"
data = jsonlib.dumps(json)
# Trigger performance measurments
perfdata = kwargs.get('perfdata', None)
if perfdata is not None:
out_headers[PERFDATA_HEADER] = 'enabled'
# Explicitly keep or close the connection
if 'Connection' not in out_headers:
out_headers['Connection'] = self.connection
out_kwargs['headers'] = out_headers
out_kwargs['body'] = data
# Add query string
if params:
out_param = []
for k, v in params.items():
if v is not None:
if isinstance(v, unicode):
v = unicode(v).encode('utf-8')
out_param.append((k, v))
encoded_args = urlencode(out_param)
url += '?' + encoded_args
if not pool_manager:
pool_manager = self.pool_manager
try:
if perfdata is not None:
request_start = monotonic_time()
resp = pool_manager.request(method, url, **out_kwargs)
if perfdata is not None:
request_end = monotonic_time()
service_perfdata = perfdata.setdefault(
self.service_type, dict())
service_perfdata['total'] = service_perfdata.get(
'total', 0.0) + request_end - request_start
body = resp.data
if body:
try:
body = jsonlib.loads(body)
except ValueError:
pass
if perfdata is not None and PERFDATA_HEADER in resp.headers:
service_perfdata = perfdata[self.service_type]
for header_val in resp.headers[PERFDATA_HEADER].split(','):
kv = header_val.split('=', 1)
service_perfdata[kv[0]] = service_perfdata.get(
kv[0], 0.0) + float(kv[1]) / 1000000.0
except urllib3.exceptions.HTTPError as exc:
oio_exception_from_httperror(exc,
reqid=out_headers.get(REQID_HEADER),
url=url)
if resp.status >= 400:
raise exceptions.from_response(resp, body)
return resp, body
def chunk_head(self, url, **kwargs):
resp = self.session.head(url)
if resp.status_code == 200:
return extract_headers_meta(resp.headers)
else:
raise exc.from_response(resp)
def _direct_request(self,
method,
url,
headers=None,
data=None,
json=None,
params=None,
admin_mode=False,
pool_manager=None,
**kwargs):
"""
Make an HTTP request.
:param method: HTTP method to use (e.g. "GET")
:type method: `str`
:param url: URL to request
:type url: `str`
:keyword admin_mode: allow operations on slave or worm namespaces
:type admin_mode: `bool`
:keyword deadline: deadline for the request, in monotonic time.
Supersedes `read_timeout`.
:type deadline: `float` seconds
:keyword timeout: optional timeout for the request (in seconds).
May be a `urllib3.Timeout(connect=connection_timeout,
read=read_timeout)`.
This method also accepts `connection_timeout` and `read_timeout`
as separate arguments.
:type timeout: `float` or `urllib3.Timeout`
:keyword headers: optional headers to add to the request
:type headers: `dict`
:raise oio.common.exceptions.OioTimeout: in case of read, write
or connection timeout
:raise oio.common.exceptions.OioNetworkException: in case of
connection error
:raise oio.common.exceptions.OioException: in other case of HTTP error
:raise oio.common.exceptions.ClientException: in case of HTTP status
code >= 400
"""
# Filter arguments that are not recognized by Requests
out_kwargs = {
k: v
for k, v in iteritems(kwargs) if k in URLLIB3_REQUESTS_KWARGS
}
# Ensure headers are all strings
if headers:
out_headers = {k: text_type(v) for k, v in headers.items()}
else:
out_headers = dict()
if self.admin_mode or admin_mode:
out_headers[ADMIN_HEADER] = '1'
# Look for a request deadline, deduce the timeout from it.
if kwargs.get('deadline', None) is not None:
to = deadline_to_timeout(kwargs['deadline'], True)
to = min(to, kwargs.get('read_timeout', to))
out_kwargs['timeout'] = urllib3.Timeout(connect=kwargs.get(
'connection_timeout', CONNECTION_TIMEOUT),
read=to)
# Shorten the deadline by 1% to compensate for the time spent
# connecting and reading response.
out_headers[TIMEOUT_HEADER] = int(to * 990000.0)
# Ensure there is a timeout
if 'timeout' not in out_kwargs:
out_kwargs['timeout'] = urllib3.Timeout(
connect=kwargs.get('connection_timeout', CONNECTION_TIMEOUT),
read=kwargs.get('read_timeout', READ_TIMEOUT))
if TIMEOUT_HEADER not in out_headers:
to = out_kwargs['timeout']
if isinstance(to, urllib3.Timeout):
to = to.read_timeout
else:
to = float(to)
out_headers[TIMEOUT_HEADER] = int(to * 1000000.0)
# Convert json and add Content-Type
if json:
out_headers["Content-Type"] = "application/json"
data = jsonlib.dumps(json)
# Trigger performance measurments
perfdata = kwargs.get('perfdata', self.perfdata)
if perfdata is not None:
out_headers[PERFDATA_HEADER] = 'enabled'
out_kwargs['headers'] = out_headers
out_kwargs['body'] = data
# Add query string
if params:
out_param = []
for k, v in params.items():
if v is not None:
if isinstance(v, text_type):
v = text_type(v).encode('utf-8')
out_param.append((k, v))
encoded_args = urlencode(out_param)
url += '?' + encoded_args
if not pool_manager:
pool_manager = self.pool_manager
def _reraise(exc_type, exc_value):
reqid = out_headers.get('X-oio-req-id')
exceptions.reraise(exc_type, exc_value, "reqid=%s" % reqid)
try:
resp = pool_manager.request(method, url, **out_kwargs)
body = resp.data
if body:
try:
body = jsonlib.loads(body)
except ValueError:
pass
if perfdata is not None and PERFDATA_HEADER in resp.headers:
for header_val in resp.headers[PERFDATA_HEADER].split(','):
kv = header_val.split('=', 1)
pdat = perfdata.get(kv[0], 0.0) + float(kv[1]) / 1000000.0
perfdata[kv[0]] = pdat
except MaxRetryError as exc:
if isinstance(exc.reason, NewConnectionError):
_reraise(exceptions.OioNetworkException, exc)
if isinstance(exc.reason, TimeoutError):
_reraise(exceptions.OioTimeout, exc)
_reraise(exceptions.OioNetworkException, exc)
except (ProtocolError, ProxyError, ClosedPoolError) as exc:
_reraise(exceptions.OioNetworkException, exc)
except TimeoutError as exc:
_reraise(exceptions.OioTimeout, exc)
except HTTPError as exc:
_reraise(exceptions.OioException, exc)
if resp.status >= 400:
raise exceptions.from_response(resp, body)
return resp, body
def chunk_delete(self, url, **kwargs):
resp = self.session.delete(url)
if resp.status_code != 204:
raise exc.from_response(resp)
def _direct_request(self,
method,
url,
headers=None,
data=None,
json=None,
params=None,
admin_mode=False,
pool_manager=None,
**kwargs):
"""
Make an HTTP request.
:param method: HTTP method to use (e.g. "GET")
:type method: `str`
:param url: URL to request
:type url: `str`
:keyword admin_mode: allow operations on slave or worm namespaces
:type admin_mode: `bool`
:keyword timeout: optional timeout for the request (in seconds).
May be a `urllib3.Timeout(connect=connection_timeout,
read=read_timeout)`.
This method also accepts `connection_timeout` and `read_timeout`
as separate arguments.
:type timeout: `float` or `urllib3.Timeout`
:keyword headers: optional headers to add to the request
:type headers: `dict`
:raise oio.common.exceptions.OioTimeout: in case of read, write
or connection timeout
:raise oio.common.exceptions.OioNetworkException: in case of
connection error
:raise oio.common.exceptions.OioException: in other case of HTTP error
:raise oio.common.exceptions.ClientException: in case of HTTP status
code >= 400
"""
# Filter arguments that are not recognized by Requests
out_kwargs = {
k: v
for k, v in kwargs.items() if k in URLLIB3_REQUESTS_KWARGS
}
# Ensure headers are all strings
if headers:
out_headers = {k: str(v) for k, v in headers.items()}
else:
out_headers = dict()
if self.admin_mode or admin_mode:
out_headers[ADMIN_HEADER] = '1'
# Ensure there is a timeout
if 'timeout' not in out_kwargs:
out_kwargs['timeout'] = urllib3.Timeout(
connect=kwargs.get('connection_timeout', CONNECTION_TIMEOUT),
read=kwargs.get('read_timeout', READ_TIMEOUT))
# Convert json and add Content-Type
if json:
out_headers["Content-Type"] = "application/json"
data = jsonlib.dumps(json)
out_kwargs['headers'] = out_headers
out_kwargs['body'] = data
# Add query string
if params:
out_param = []
for k, v in params.items():
if v is not None:
if isinstance(v, unicode):
v = unicode(v).encode('utf-8')
out_param.append((k, v))
encoded_args = urlencode(out_param)
url += '?' + encoded_args
if not pool_manager:
pool_manager = self.pool_manager
try:
resp = pool_manager.request(method, url, **out_kwargs)
body = resp.data
if body:
try:
body = jsonlib.loads(body)
except ValueError:
pass
except MaxRetryError as exc:
if isinstance(exc.reason, NewConnectionError):
raise exceptions.OioNetworkException(exc), None, \
sys.exc_info()[2]
if isinstance(exc.reason, TimeoutError):
raise exceptions.OioTimeout(exc), None, sys.exc_info()[2]
raise exceptions.OioNetworkException(exc), None, sys.exc_info()[2]
except (ProtocolError, ProxyError, ClosedPoolError) as exc:
raise exceptions.OioNetworkException(exc), None, sys.exc_info()[2]
except TimeoutError as exc:
raise exceptions.OioTimeout(exc), None, sys.exc_info()[2]
except HTTPError as exc:
raise exceptions.OioException(exc), None, sys.exc_info()[2]
if resp.status >= 400:
raise exceptions.from_response(resp, body)
return resp, body
def chunk_delete(self, url, **kwargs):
resp = self.http_pool.request('DELETE', self.resolve_url(url),
**kwargs)
if resp.status != 204:
raise exc.from_response(resp)
return resp
def chunk_head(self, url, **kwargs):
resp = self.session.head(url)
if resp.status_code == 200:
return extract_headers_meta(resp.headers)
else:
raise exc.from_response(resp)
