def test_deadline_to_timeout(self):
self.assertRaises(DeadlineReached, deadline_to_timeout,
monotonic_time() - 0.001, True)
self.assertRaises(DeadlineReached, deadline_to_timeout,
monotonic_time(), True)
deadline = monotonic_time() + 1.0
to = deadline_to_timeout(deadline, True)
self.assertLessEqual(to, 1.0)
self.assertGreater(to, 0.9)
def _direct_request(self,
method,
url,
headers=None,
data=None,
json=None,
params=None,
admin_mode=False,
pool_manager=None,
**kwargs):
"""
Make an HTTP request.
:param method: HTTP method to use (e.g. "GET")
:type method: `str`
:param url: URL to request
:type url: `str`
:keyword admin_mode: allow operations on slave or worm namespaces
:type admin_mode: `bool`
:keyword deadline: deadline for the request, in monotonic time.
Supersedes `read_timeout`.
:type deadline: `float` seconds
:keyword timeout: optional timeout for the request (in seconds).
May be a `urllib3.Timeout(connect=connection_timeout,
read=read_timeout)`.
This method also accepts `connection_timeout` and `read_timeout`
as separate arguments.
:type timeout: `float` or `urllib3.Timeout`
:keyword headers: optional headers to add to the request
:type headers: `dict`
:raise oio.common.exceptions.OioTimeout: in case of read, write
or connection timeout
:raise oio.common.exceptions.OioNetworkException: in case of
connection error
:raise oio.common.exceptions.OioException: in other case of HTTP error
:raise oio.common.exceptions.ClientException: in case of HTTP status
code >= 400
"""
# Filter arguments that are not recognized by Requests
out_kwargs = {
k: v
for k, v in iteritems(kwargs) if k in URLLIB3_REQUESTS_KWARGS
}
# Ensure headers are all strings
if headers:
out_headers = {k: text_type(v) for k, v in headers.items()}
else:
out_headers = dict()
if self.admin_mode or admin_mode:
out_headers[ADMIN_HEADER] = '1'
# Look for a request deadline, deduce the timeout from it.
if kwargs.get('deadline', None) is not None:
to = deadline_to_timeout(kwargs['deadline'], True)
to = min(to, kwargs.get('read_timeout', to))
out_kwargs['timeout'] = urllib3.Timeout(connect=kwargs.get(
'connection_timeout', CONNECTION_TIMEOUT),
read=to)
# Shorten the deadline by 1% to compensate for the time spent
# connecting and reading response.
out_headers[TIMEOUT_HEADER] = int(to * 990000.0)
# Ensure there is a timeout
if 'timeout' not in out_kwargs:
out_kwargs['timeout'] = urllib3.Timeout(
connect=kwargs.get('connection_timeout', CONNECTION_TIMEOUT),
read=kwargs.get('read_timeout', READ_TIMEOUT))
if TIMEOUT_HEADER not in out_headers:
to = out_kwargs['timeout']
if isinstance(to, urllib3.Timeout):
to = to.read_timeout
else:
to = float(to)
out_headers[TIMEOUT_HEADER] = int(to * 1000000.0)
# Convert json and add Content-Type
if json:
out_headers["Content-Type"] = "application/json"
data = jsonlib.dumps(json)
# Trigger performance measurments
perfdata = kwargs.get('perfdata', self.perfdata)
if perfdata is not None:
out_headers[PERFDATA_HEADER] = 'enabled'
out_kwargs['headers'] = out_headers
out_kwargs['body'] = data
# Add query string
if params:
out_param = []
for k, v in params.items():
if v is not None:
if isinstance(v, text_type):
v = text_type(v).encode('utf-8')
out_param.append((k, v))
encoded_args = urlencode(out_param)
url += '?' + encoded_args
if not pool_manager:
pool_manager = self.pool_manager
def _reraise(exc_type, exc_value):
reqid = out_headers.get('X-oio-req-id')
exceptions.reraise(exc_type, exc_value, "reqid=%s" % reqid)
try:
resp = pool_manager.request(method, url, **out_kwargs)
body = resp.data
if body:
try:
body = jsonlib.loads(body)
except ValueError:
pass
if perfdata is not None and PERFDATA_HEADER in resp.headers:
for header_val in resp.headers[PERFDATA_HEADER].split(','):
kv = header_val.split('=', 1)
pdat = perfdata.get(kv[0], 0.0) + float(kv[1]) / 1000000.0
perfdata[kv[0]] = pdat
except MaxRetryError as exc:
if isinstance(exc.reason, NewConnectionError):
_reraise(exceptions.OioNetworkException, exc)
if isinstance(exc.reason, TimeoutError):
_reraise(exceptions.OioTimeout, exc)
_reraise(exceptions.OioNetworkException, exc)
except (ProtocolError, ProxyError, ClosedPoolError) as exc:
_reraise(exceptions.OioNetworkException, exc)
except TimeoutError as exc:
_reraise(exceptions.OioTimeout, exc)
except HTTPError as exc:
_reraise(exceptions.OioException, exc)
if resp.status >= 400:
raise exceptions.from_response(resp, body)
return resp, body
def write_timeout(self):
if self.deadline is None:
return self._write_timeout
dl_to = deadline_to_timeout(self.deadline, True)
return min(dl_to, self._write_timeout)
def read_timeout(self):
if 'read_timeout' in self.extra_kwargs:
return self.extra_kwargs['read_timeout']
elif self.deadline is not None:
return deadline_to_timeout(self.deadline, True)
return CLIENT_TIMEOUT
def write_timeout(self):
if 'write_timeout' in self.extra_kwargs:
return self.extra_kwargs['write_timeout']
elif self.deadline is not None:
return deadline_to_timeout(self.deadline, True)
return CHUNK_TIMEOUT
def _direct_request(self, method, url, headers=None, data=None, json=None,
params=None, admin_mode=False, pool_manager=None,
force_master=False, **kwargs):
"""
Make an HTTP request.
:param method: HTTP method to use (e.g. "GET")
:type method: `str`
:param url: URL to request
:type url: `str`
:keyword admin_mode: allow operations on slave or worm namespaces
:type admin_mode: `bool`
:keyword deadline: deadline for the request, in monotonic time.
Supersedes `read_timeout`.
:type deadline: `float` seconds
:keyword timeout: optional timeout for the request (in seconds).
May be a `urllib3.Timeout(connect=connection_timeout,
read=read_timeout)`.
This method also accepts `connection_timeout` and `read_timeout`
as separate arguments.
:type timeout: `float` or `urllib3.Timeout`
:keyword headers: optional headers to add to the request
:type headers: `dict`
:keyword force_master: request will run on master service only.
:type force_master: `bool`
:raise oio.common.exceptions.OioTimeout: in case of read, write
or connection timeout
:raise oio.common.exceptions.OioNetworkException: in case of
connection error
:raise oio.common.exceptions.OioException: in other case of HTTP error
:raise oio.common.exceptions.ClientException: in case of HTTP status
code >= 400
"""
out_kwargs = dict(kwargs)
# Ensure headers are all strings
if headers:
out_headers = {k: str(v) for k, v in headers.items()}
else:
out_headers = dict()
if self.admin_mode or admin_mode:
out_headers[ADMIN_HEADER] = '1'
if self.force_master or force_master:
out_headers[FORCEMASTER_HEADER] = '1'
# Look for a request deadline, deduce the timeout from it.
if kwargs.get('deadline', None) is not None:
to = deadline_to_timeout(kwargs['deadline'], True)
to = min(to, kwargs.get('read_timeout', to))
out_kwargs['timeout'] = urllib3.Timeout(
connect=kwargs.get('connection_timeout', CONNECTION_TIMEOUT),
read=to)
# Shorten the deadline by 1% to compensate for the time spent
# connecting and reading response.
out_headers[TIMEOUT_HEADER] = int(to * 990000.0)
# Ensure there is a timeout
if 'timeout' not in out_kwargs:
out_kwargs['timeout'] = urllib3.Timeout(
connect=kwargs.get('connection_timeout', CONNECTION_TIMEOUT),
read=kwargs.get('read_timeout', READ_TIMEOUT))
if TIMEOUT_HEADER not in out_headers:
to = out_kwargs['timeout']
if isinstance(to, urllib3.Timeout):
to = to.read_timeout
else:
to = float(to)
out_headers[TIMEOUT_HEADER] = int(to * 1000000.0)
# Look for a request ID
if 'reqid' in kwargs:
out_headers[REQID_HEADER] = str(kwargs['reqid'])
if len(out_headers.get(REQID_HEADER, '')) > STRLEN_REQID:
out_headers[REQID_HEADER] = \
out_headers[REQID_HEADER][:STRLEN_REQID]
self.__logger().warn('Request ID truncated to %d characters',
STRLEN_REQID)
# Convert json and add Content-Type
if json:
out_headers["Content-Type"] = "application/json"
data = jsonlib.dumps(json)
# Trigger performance measurments
perfdata = kwargs.get('perfdata', None)
if perfdata is not None:
out_headers[PERFDATA_HEADER] = 'enabled'
# Explicitly keep or close the connection
if 'Connection' not in out_headers:
out_headers['Connection'] = self.connection
out_kwargs['headers'] = out_headers
out_kwargs['body'] = data
# Add query string
if params:
out_param = []
for k, v in params.items():
if v is not None:
if isinstance(v, unicode):
v = unicode(v).encode('utf-8')
out_param.append((k, v))
encoded_args = urlencode(out_param)
url += '?' + encoded_args
if not pool_manager:
pool_manager = self.pool_manager
try:
if perfdata is not None:
request_start = monotonic_time()
resp = pool_manager.request(method, url, **out_kwargs)
if perfdata is not None:
request_end = monotonic_time()
service_perfdata = perfdata.setdefault(
self.service_type, dict())
service_perfdata['total'] = service_perfdata.get(
'total', 0.0) + request_end - request_start
body = resp.data
if body:
try:
body = jsonlib.loads(body)
except ValueError:
pass
if perfdata is not None and PERFDATA_HEADER in resp.headers:
service_perfdata = perfdata[self.service_type]
for header_val in resp.headers[PERFDATA_HEADER].split(','):
kv = header_val.split('=', 1)
service_perfdata[kv[0]] = service_perfdata.get(
kv[0], 0.0) + float(kv[1]) / 1000000.0
except urllib3.exceptions.HTTPError as exc:
oio_exception_from_httperror(exc,
reqid=out_headers.get(REQID_HEADER),
url=url)
if resp.status >= 400:
raise exceptions.from_response(resp, body)
return resp, body
