def urlopen(self, method, url, redirect=True, **kw):
"""
Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen`
with custom cross-host redirect logic and only sends the request-uri
portion of the ``url``.
The given ``url`` parameter must be absolute, such that an appropriate
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
"""
#===============================================================================================================
# add by mz
error_type = kw.get('error_type')
if error_type:
from urllib3.exceptions import LocationValueError, HostChangedError, LocationParseError, ConnectTimeoutError
from urllib3.exceptions import ProxyError, TimeoutError, ReadTimeoutError, ProtocolError, DecodeError
from urllib3.exceptions import ResponseError, ResponseNotChunked, SSLError, HTTPError, HTTPWarning, PoolError
from urllib3.exceptions import RequestError, MaxRetryError, TimeoutStateError, NewConnectionError
from urllib3.exceptions import EmptyPoolError, ClosedPoolError, SecurityWarning, SubjectAltNameWarning
from urllib3.exceptions import InsecureRequestWarning, SystemTimeWarning, InsecurePlatformWarning
from urllib3.exceptions import SNIMissingWarning, DependencyWarning, ProxySchemeUnknown, HeaderParsingError
get_error = {
"LocationValueError":
LocationValueError(),
"HostChangedError":
HostChangedError(pool=1, url=2),
"LocationParseError":
LocationParseError(url),
"ConnectTimeoutError":
ConnectTimeoutError(),
"ProxyError":
ProxyError(),
"TimeoutError":
TimeoutError(),
"ReadTimeoutError":
ReadTimeoutError(pool=1, url=2, message="ReadTimeoutError"),
"ProtocolError":
ProtocolError(),
"DecodeError":
DecodeError(),
"ResponseError":
ResponseError(),
"ResponseNotChunked":
ResponseNotChunked(),
"SSLError":
SSLError(),
"HTTPError":
HTTPError(),
"HTTPWarning":
HTTPWarning(),
"PoolError":
PoolError(pool=1, message=2),
"RequestError":
RequestError(pool=1, url=2, message="RequestError"),
"MaxRetryError":
MaxRetryError(pool=1, url=2, reason=None),
"TimeoutStateError":
TimeoutStateError(),
"NewConnectionError":
NewConnectionError(pool=1, message="NewConnectionError"),
"EmptyPoolError":
EmptyPoolError(pool=1, message="EmptyPoolError"),
"ClosedPoolError":
ClosedPoolError(pool=1, message="ClosedPoolError"),
"SecurityWarning":
SecurityWarning(),
"SubjectAltNameWarning":
SubjectAltNameWarning(),
"InsecureRequestWarning":
InsecureRequestWarning(),
"SystemTimeWarning":
SystemTimeWarning(),
"InsecurePlatformWarning":
InsecurePlatformWarning(),
"SNIMissingWarning":
SNIMissingWarning(),
"DependencyWarning":
DependencyWarning(),
"ProxySchemeUnknown":
ProxySchemeUnknown(scheme=1),
"HeaderParsingError":
HeaderParsingError(defects=1, unparsed_data=2)
}
error_ = get_error[error_type]
raise error_
#===============================================================================================================
u = parse_url(url)
conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
kw['assert_same_host'] = False
kw['redirect'] = False
if 'headers' not in kw:
kw['headers'] = self.headers
if self.proxy is not None and u.scheme == "http":
response = conn.urlopen(method, url, **kw)
else:
response = conn.urlopen(method, u.request_uri, **kw)
redirect_location = redirect and response.get_redirect_location()
if not redirect_location:
return response
# Support relative URLs for redirecting.
redirect_location = urljoin(url, redirect_location)
# RFC 7231, Section 6.4.4
if response.status == 303:
method = 'GET'
retries = kw.get('retries')
if not isinstance(retries, Retry):
retries = Retry.from_int(retries, redirect=redirect)
try:
retries = retries.increment(method,
url,
response=response,
_pool=conn)
except MaxRetryError:
if retries.raise_on_redirect:
raise
return response
kw['retries'] = retries
kw['redirect'] = redirect
log.info("Redirecting %s -> %s" % (url, redirect_location))
return self.urlopen(method, redirect_location, **kw)
def execute_query(self, query, disable_log=False, isql_api=False):
"""Execute a sparql query
Parameters
----------
query : string
Query to perform
Returns
-------
TYPE
result
"""
try:
triplestore = self.settings.get("triplestore", "triplestore")
# Use ISQL or SPARQL
isql_api_url = None
try:
isql_api_url = self.settings.get("triplestore", "isqlapi")
except Exception:
pass
use_isql = True if triplestore == "virtuoso" and isql_api_url and self.local_query and isql_api else False
start_time = time.time()
self.endpoint.setQuery(query)
# Debug
if self.settings.getboolean('askomics', 'debug'):
self.log.debug("Launch {} query on {} ({})".format(
"ISQL" if use_isql else "SPARQL", self.triplestore,
self.url_endpoint))
self.log.debug(query)
if use_isql:
formatted_query = "SPARQL {}".format(query)
json = {
"command": formatted_query,
"disable_log": disable_log,
"sparql_select": not self.endpoint.isSparqlUpdateRequest()
}
response = requests.post(url=isql_api_url, json=json)
results = response.json()
if results["status"] == 500:
raise HTTPError("isqlapi: {}".format(results["message"]))
else:
# Update
if self.endpoint.isSparqlUpdateRequest():
self.endpoint.setMethod('POST')
# Virtuoso hack
if self.triplestore == 'virtuoso':
self.endpoint.queryType = "SELECT"
results = self.endpoint.query()
# Select
else:
self.endpoint.setReturnFormat(JSON)
results = self.endpoint.query().convert()
self.query_time = time.time() - start_time
return results
except Exception as e:
traceback.print_exc(file=sys.stdout)
raise type(e)("Triplestore error: {}".format(
str(e))).with_traceback(sys.exc_info()[2])
def test_broken_response(self):
request = self.factory.get('/')
urlopen_mock = MagicMock(side_effect=HTTPError())
with patch(URLOPEN, urlopen_mock), self.assertRaises(HTTPError):
CustomProxyView.as_view()(request, path='/')
def test_exceptions(self):
assert self.verify_pickling(HTTPError(None))
assert self.verify_pickling(MaxRetryError(None, None, None))
assert self.verify_pickling(LocationParseError(None))
assert self.verify_pickling(ConnectTimeoutError(None))
def test_exceptions(self):
assert pickle.dumps(HTTPError(None))
assert pickle.dumps(MaxRetryError(None, None))
assert pickle.dumps(LocationParseError(None))
def test_exceptions_with_objects(self):
assert pickle.dumps(HTTPError('foo'))
assert pickle.dumps(MaxRetryError(HTTPConnectionPool('localhost'),
'/'))
assert pickle.dumps(LocationParseError('fake location'))
def test_exceptions(self):
assert self.cycle(HTTPError(None))
assert self.cycle(MaxRetryError(None, None, None))
assert self.cycle(LocationParseError(None))
def test_directory_deletion(self):
self._run_initial_resync()
watcher_req = self.watcher_etcd.get_next_request()
with patch("calico.etcddriver.driver.monotonic_time",
autospec=True) as m_mon:
# The watcher has code to detect tight loops, vary the duration
# between timeouts/exceptions so that we trigger that on the first
# loop.
m_mon.side_effect = iter([
0.1, # ReadTimeoutError req_end_time
0.1, # req_start_time
0.2, # HTTPError req_end_time
0.2, # req_start_time
30, # HTTPException req_end_time
40, # req_start_time
50, # socket.error req_end_time
])
# For coverage: Nothing happens for a while, poll times out.
watcher_req.respond_with_exception(ReadTimeoutError(
Mock(), "", ""))
with patch("time.sleep", autospec=True) as m_sleep:
for exc in [HTTPError(), HTTPException(), socket.error()]:
# For coverage: non-timeout errors.
watcher_req = self.watcher_etcd.get_next_request()
watcher_req.respond_with_exception(exc)
self.assertEqual(m_sleep.mock_calls, [call(0.1)])
# For coverage: Then a set to a dir, which should be ignored.
watcher_req = self.watcher_etcd.get_next_request()
watcher_req.respond_with_data(
json.dumps({
"action": "create",
"node": {
"key": "/calico/v1/foo",
"dir": True,
"modifiedIndex": 100,
}
}), 100, 200)
# Then a whole directory is deleted.
watcher_req = self.watcher_etcd.assert_request(
VERSION_DIR,
timeout=90,
recursive=True,
wait_index=101,
)
watcher_req.respond_with_value(
"/calico/v1/adir",
dir=True,
value=None,
action="delete",
mod_index=101,
status=300 # For coverage of warning log.
)
# Should get individual deletes for each one then a flush. We're
# relying on the trie returning sorted results here.
self.assert_msg_to_felix(MSG_TYPE_UPDATE, {
MSG_KEY_KEY: "/calico/v1/adir/akey",
MSG_KEY_VALUE: None,
})
self.assert_msg_to_felix(MSG_TYPE_UPDATE, {
MSG_KEY_KEY: "/calico/v1/adir/bkey",
MSG_KEY_VALUE: None,
})
self.assert_msg_to_felix(MSG_TYPE_UPDATE, {
MSG_KEY_KEY: "/calico/v1/adir/ckey",
MSG_KEY_VALUE: None,
})
self.assert_msg_to_felix(MSG_TYPE_UPDATE, {
MSG_KEY_KEY: "/calico/v1/adir/ekey",
MSG_KEY_VALUE: None,
})
self.assert_flush_to_felix()
# Check the contents of the trie.
keys = set(self.driver._hwms._hwms.keys())
self.assertEqual(
keys, set([u'/calico/v1/Ready/', u'/calico/v1/adir2/dkey/']))
def raise_for_status(res: HTTPResponse) -> None:
if status.is_client_error(res.status) or status.is_server_error(
res.status):
raise HTTPError(f"{res.status}:{res.reason}")
