def urlopen(self, method, url, redirect=True, **kw): """ Same as :meth:`urllib3.connectionpool.HTTPConnectionPool.urlopen` with custom cross-host redirect logic and only sends the request-uri portion of the ``url``. The given ``url`` parameter must be absolute, such that an appropriate :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it. """ #=============================================================================================================== # add by mz error_type = kw.get('error_type') if error_type: from urllib3.exceptions import LocationValueError, HostChangedError, LocationParseError, ConnectTimeoutError from urllib3.exceptions import ProxyError, TimeoutError, ReadTimeoutError, ProtocolError, DecodeError from urllib3.exceptions import ResponseError, ResponseNotChunked, SSLError, HTTPError, HTTPWarning, PoolError from urllib3.exceptions import RequestError, MaxRetryError, TimeoutStateError, NewConnectionError from urllib3.exceptions import EmptyPoolError, ClosedPoolError, SecurityWarning, SubjectAltNameWarning from urllib3.exceptions import InsecureRequestWarning, SystemTimeWarning, InsecurePlatformWarning from urllib3.exceptions import SNIMissingWarning, DependencyWarning, ProxySchemeUnknown, HeaderParsingError get_error = { "LocationValueError": LocationValueError(), "HostChangedError": HostChangedError(pool=1, url=2), "LocationParseError": LocationParseError(url), "ConnectTimeoutError": ConnectTimeoutError(), "ProxyError": ProxyError(), "TimeoutError": TimeoutError(), "ReadTimeoutError": ReadTimeoutError(pool=1, url=2, message="ReadTimeoutError"), "ProtocolError": ProtocolError(), "DecodeError": DecodeError(), "ResponseError": ResponseError(), "ResponseNotChunked": ResponseNotChunked(), "SSLError": SSLError(), "HTTPError": HTTPError(), "HTTPWarning": HTTPWarning(), "PoolError": PoolError(pool=1, message=2), "RequestError": RequestError(pool=1, url=2, message="RequestError"), "MaxRetryError": MaxRetryError(pool=1, url=2, reason=None), "TimeoutStateError": TimeoutStateError(), "NewConnectionError": NewConnectionError(pool=1, message="NewConnectionError"), "EmptyPoolError": EmptyPoolError(pool=1, message="EmptyPoolError"), "ClosedPoolError": ClosedPoolError(pool=1, message="ClosedPoolError"), "SecurityWarning": SecurityWarning(), "SubjectAltNameWarning": SubjectAltNameWarning(), "InsecureRequestWarning": InsecureRequestWarning(), "SystemTimeWarning": SystemTimeWarning(), "InsecurePlatformWarning": InsecurePlatformWarning(), "SNIMissingWarning": SNIMissingWarning(), "DependencyWarning": DependencyWarning(), "ProxySchemeUnknown": ProxySchemeUnknown(scheme=1), "HeaderParsingError": HeaderParsingError(defects=1, unparsed_data=2) } error_ = get_error[error_type] raise error_ #=============================================================================================================== u = parse_url(url) conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme) kw['assert_same_host'] = False kw['redirect'] = False if 'headers' not in kw: kw['headers'] = self.headers if self.proxy is not None and u.scheme == "http": response = conn.urlopen(method, url, **kw) else: response = conn.urlopen(method, u.request_uri, **kw) redirect_location = redirect and response.get_redirect_location() if not redirect_location: return response # Support relative URLs for redirecting. redirect_location = urljoin(url, redirect_location) # RFC 7231, Section 6.4.4 if response.status == 303: method = 'GET' retries = kw.get('retries') if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect) try: retries = retries.increment(method, url, response=response, _pool=conn) except MaxRetryError: if retries.raise_on_redirect: raise return response kw['retries'] = retries kw['redirect'] = redirect log.info("Redirecting %s -> %s" % (url, redirect_location)) return self.urlopen(method, redirect_location, **kw)
def execute_query(self, query, disable_log=False, isql_api=False): """Execute a sparql query Parameters ---------- query : string Query to perform Returns ------- TYPE result """ try: triplestore = self.settings.get("triplestore", "triplestore") # Use ISQL or SPARQL isql_api_url = None try: isql_api_url = self.settings.get("triplestore", "isqlapi") except Exception: pass use_isql = True if triplestore == "virtuoso" and isql_api_url and self.local_query and isql_api else False start_time = time.time() self.endpoint.setQuery(query) # Debug if self.settings.getboolean('askomics', 'debug'): self.log.debug("Launch {} query on {} ({})".format( "ISQL" if use_isql else "SPARQL", self.triplestore, self.url_endpoint)) self.log.debug(query) if use_isql: formatted_query = "SPARQL {}".format(query) json = { "command": formatted_query, "disable_log": disable_log, "sparql_select": not self.endpoint.isSparqlUpdateRequest() } response = requests.post(url=isql_api_url, json=json) results = response.json() if results["status"] == 500: raise HTTPError("isqlapi: {}".format(results["message"])) else: # Update if self.endpoint.isSparqlUpdateRequest(): self.endpoint.setMethod('POST') # Virtuoso hack if self.triplestore == 'virtuoso': self.endpoint.queryType = "SELECT" results = self.endpoint.query() # Select else: self.endpoint.setReturnFormat(JSON) results = self.endpoint.query().convert() self.query_time = time.time() - start_time return results except Exception as e: traceback.print_exc(file=sys.stdout) raise type(e)("Triplestore error: {}".format( str(e))).with_traceback(sys.exc_info()[2])
def test_broken_response(self): request = self.factory.get('/') urlopen_mock = MagicMock(side_effect=HTTPError()) with patch(URLOPEN, urlopen_mock), self.assertRaises(HTTPError): CustomProxyView.as_view()(request, path='/')
def test_exceptions(self): assert self.verify_pickling(HTTPError(None)) assert self.verify_pickling(MaxRetryError(None, None, None)) assert self.verify_pickling(LocationParseError(None)) assert self.verify_pickling(ConnectTimeoutError(None))
def test_exceptions(self): assert pickle.dumps(HTTPError(None)) assert pickle.dumps(MaxRetryError(None, None)) assert pickle.dumps(LocationParseError(None))
def test_exceptions_with_objects(self): assert pickle.dumps(HTTPError('foo')) assert pickle.dumps(MaxRetryError(HTTPConnectionPool('localhost'), '/')) assert pickle.dumps(LocationParseError('fake location'))
def test_exceptions(self): assert self.cycle(HTTPError(None)) assert self.cycle(MaxRetryError(None, None, None)) assert self.cycle(LocationParseError(None))
def test_directory_deletion(self): self._run_initial_resync() watcher_req = self.watcher_etcd.get_next_request() with patch("calico.etcddriver.driver.monotonic_time", autospec=True) as m_mon: # The watcher has code to detect tight loops, vary the duration # between timeouts/exceptions so that we trigger that on the first # loop. m_mon.side_effect = iter([ 0.1, # ReadTimeoutError req_end_time 0.1, # req_start_time 0.2, # HTTPError req_end_time 0.2, # req_start_time 30, # HTTPException req_end_time 40, # req_start_time 50, # socket.error req_end_time ]) # For coverage: Nothing happens for a while, poll times out. watcher_req.respond_with_exception(ReadTimeoutError( Mock(), "", "")) with patch("time.sleep", autospec=True) as m_sleep: for exc in [HTTPError(), HTTPException(), socket.error()]: # For coverage: non-timeout errors. watcher_req = self.watcher_etcd.get_next_request() watcher_req.respond_with_exception(exc) self.assertEqual(m_sleep.mock_calls, [call(0.1)]) # For coverage: Then a set to a dir, which should be ignored. watcher_req = self.watcher_etcd.get_next_request() watcher_req.respond_with_data( json.dumps({ "action": "create", "node": { "key": "/calico/v1/foo", "dir": True, "modifiedIndex": 100, } }), 100, 200) # Then a whole directory is deleted. watcher_req = self.watcher_etcd.assert_request( VERSION_DIR, timeout=90, recursive=True, wait_index=101, ) watcher_req.respond_with_value( "/calico/v1/adir", dir=True, value=None, action="delete", mod_index=101, status=300 # For coverage of warning log. ) # Should get individual deletes for each one then a flush. We're # relying on the trie returning sorted results here. self.assert_msg_to_felix(MSG_TYPE_UPDATE, { MSG_KEY_KEY: "/calico/v1/adir/akey", MSG_KEY_VALUE: None, }) self.assert_msg_to_felix(MSG_TYPE_UPDATE, { MSG_KEY_KEY: "/calico/v1/adir/bkey", MSG_KEY_VALUE: None, }) self.assert_msg_to_felix(MSG_TYPE_UPDATE, { MSG_KEY_KEY: "/calico/v1/adir/ckey", MSG_KEY_VALUE: None, }) self.assert_msg_to_felix(MSG_TYPE_UPDATE, { MSG_KEY_KEY: "/calico/v1/adir/ekey", MSG_KEY_VALUE: None, }) self.assert_flush_to_felix() # Check the contents of the trie. keys = set(self.driver._hwms._hwms.keys()) self.assertEqual( keys, set([u'/calico/v1/Ready/', u'/calico/v1/adir2/dkey/']))
def raise_for_status(res: HTTPResponse) -> None: if status.is_client_error(res.status) or status.is_server_error( res.status): raise HTTPError(f"{res.status}:{res.reason}")